yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-24 14:08:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add DDL query ingest (#15860)

Browse Source
This commit is contained in:
Suman Maharana 2024-05-06 18:03:50 +05:30 committed by GitHub
parent 2792bbd288
commit 488078da8a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
73 changed files with 441 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
bootstrap/sql/migrations/native/1.4.0/mysql/schemaChanges.sql
View File

@ -273,3 +273,11 @@ SET json = JSON_INSERT(
where serviceType = 'OpenLineage'
AND JSON_EXTRACT(json, '$.connection.config.SSLCALocation') IS NOT NULL;
-- Change viewDefinition to schemaDefinition
UPDATE table_entity
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.viewDefinition'),
'$.schemaDefinition',
JSON_EXTRACT(json, '$.viewDefinition')
);

9
bootstrap/sql/migrations/native/1.4.0/postgres/schemaChanges.sql
View File

@ -269,3 +269,12 @@ SET json = jsonb_set(
json#>'{connection,config,connection,sslConfig}' || jsonb_build_object('caCertificate', json#>'{connection,config,connection,SSLCALocation}')
)
WHERE serviceType IN ('OpenLineage') AND json#>'{connection,config,connection,SSLCALocation}' IS NOT NULL;
-- Change viewDefinition to schemaDefinition
UPDATE table_entity
SET json = jsonb_set(
json::jsonb,
'{schemaDefinition}',
json->'viewDefinition'
) - 'viewDefinition'
WHERE jsonb_exists(json::jsonb, 'viewDefinition') = true;

9
ingestion/examples/sample_data/datasets/tables.json
View File

@ -8,6 +8,7 @@
"updatedBy": "anonymous",
"href": "http://localhost:8585/api/v1/tables/3cda8ecb-f4c6-4ed4-8506-abe965b54b86",
"tableType": "Regular",
"schemaDefinition": "CREATE TABLE dim_address(address_id NUMERIC PRIMARY KEY, shop_id NUMERIC)",
"retentionPeriod": "10D",
"columns": [
{
@ -734,7 +735,8 @@
"updatedAt": 1638354087591,
"updatedBy": "anonymous",
"href": "http://localhost:8585/api/v1/tables/1cda9ecb-f4c6-4ed4-8506-abe965b64c87",
"tableType": "Regular",
"tableType": "View",
"schemaDefinition": "CREATE VIEW dim_address_clean(address_id NUMERIC PRIMARY KEY, shop_id NUMERIC)",
"columns": [
{
"name": "address_id",
@ -1069,6 +1071,7 @@
"updatedBy": "anonymous",
"href": "http://localhost:8585/api/v1/tables/753833e0-f526-47b6-8555-97f5bb2882d5",
"tableType": "Regular",
"schemaDefinition": "CREATE TABLE dim.api/client(api_client_id NUMERIC PRIMARY KEY, title VARCHAR)",
"columns": [
{
"name": "api_client_id",
@ -1328,6 +1331,7 @@
"updatedBy": "anonymous",
"href": "http://localhost:8585/api/v1/tables/9c96042f-311a-4826-b5b2-8658914e2ee8",
"tableType": "Regular",
"schemaDefinition": "CREATE TABLE dim_customer(customer_id NUMERIC PRIMARY KEY, shop_id NUMERIC, average_order_size NUMERIC)",
"columns": [
{
"name": "customer_id",
@ -2782,6 +2786,7 @@
"updatedBy": "anonymous",
"href": "http://localhost:8585/api/v1/tables/e64426b9-852a-468e-ac16-1231bb01fe96",
"tableType": "Regular",
"schemaDefinition": "CREATE TABLE dim_location(location_id NUMERIC PRIMARY KEY, shop_id NUMERIC)",
"columns": [
{
"name": "location_id",
@ -3109,6 +3114,7 @@
"updatedBy": "anonymous",
"href": "http://localhost:8585/api/v1/tables/f888bde6-ce08-42c3-96aa-1d726e248930",
"tableType": "Regular",
"schemaDefinition": "CREATE TABLE dim.produc(product_id NUMERIC PRIMARY KEY, shop_id NUMERIC)",
"columns": [
{
"name": "product_id",
@ -3494,6 +3500,7 @@
"updatedBy": "anonymous",
"href": "http://localhost:8585/api/v1/tables/c71690a3-0764-4791-a1d3-0c47f1e0c2ab",
"tableType": "Regular",
"schemaDefinition": "CREATE TABLE dim.product.variant(product_variant_id NUMERIC PRIMARY KEY, shop_id NUMERIC)",
"columns": [
{
"name": "product_variant_id",

2
ingestion/src/metadata/ingestion/models/patch_request.py
View File

@ -88,7 +88,7 @@ ALLOWED_COMMON_PATCH_FIELDS = {
"tableConstraints": True,
"tablePartition": True,
"location": True,
"viewDefinition": True,
"schemaDefinition": True,
"sampleData": True,
"fileFormat": True,
# Stored Procedure Fields

37
ingestion/src/metadata/ingestion/source/database/bigquery/metadata.py
View File

@ -99,7 +99,11 @@ from metadata.utils.credentials import GOOGLE_CREDENTIALS
from metadata.utils.filters import filter_by_database, filter_by_schema
from metadata.utils.helpers import get_start_and_end
from metadata.utils.logger import ingestion_logger
from metadata.utils.sqlalchemy_utils import is_complex_type
from metadata.utils.sqlalchemy_utils import (
get_all_table_ddls,
get_table_ddl,
is_complex_type,
)
from metadata.utils.tag_utils import get_ometa_tag_and_classification, get_tag_label
from metadata.utils.tag_utils import get_tag_labels as fetch_tag_labels_om
@ -210,6 +214,9 @@ BigQueryDialect._build_formatted_table_id = ( # pylint: disable=protected-acces
BigQueryDialect.get_pk_constraint = get_pk_constraint
BigQueryDialect.get_foreign_keys = get_foreign_keys
Inspector.get_all_table_ddls = get_all_table_ddls
Inspector.get_table_ddl = get_table_ddl
class BigquerySource(
LifeCycleQueryMixin, StoredProcedureMixin, CommonDbSourceService, MultiDBSource
@ -601,21 +608,33 @@ class BigquerySource(
f"Error trying to connect to database {project_id}: {exc}"
)
def get_view_definition(
def get_schema_definition(
self, table_type: str, table_name: str, schema_name: str, inspector: Inspector
) -> Optional[str]:
if table_type == TableType.View:
try:
"""
Get the DDL statement or View Definition for a table
"""
try:
if table_type == TableType.View:
view_definition = inspector.get_view_definition(
fqn._build(self.context.get().database, schema_name, table_name)
)
view_definition = (
"" if view_definition is None else str(view_definition)
f"CREATE VIEW {schema_name}.{table_name} AS {str(view_definition)}"
if view_definition is not None
else None
)
except NotImplementedError:
logger.warning("View definition not implemented")
view_definition = ""
return f"CREATE VIEW {schema_name}.{table_name} AS {view_definition}"
return view_definition
schema_definition = inspector.get_table_ddl(
self.connection, table_name, schema_name
)
schema_definition = (
str(schema_definition) if schema_definition is not None else None
)
return schema_definition
except NotImplementedError:
logger.warning("Schema definition not implemented")
return None
def get_table_partition_details(

40
ingestion/src/metadata/ingestion/source/database/clickhouse/metadata.py
View File

@ -45,7 +45,9 @@ from metadata.ingestion.source.database.common_db_source import (
from metadata.utils.logger import ingestion_logger
from metadata.utils.sqlalchemy_utils import (
get_all_table_comments,
get_all_table_ddls,
get_all_view_definitions,
get_table_ddl,
)
logger = ingestion_logger()
@ -95,6 +97,8 @@ ClickHouseDialect._get_column_info = ( # pylint: disable=protected-access
)
Inspector.get_mview_names = get_mview_names
ClickHouseDialect.get_mview_names = get_mview_names_dialect
Inspector.get_all_table_ddls = get_all_table_ddls
Inspector.get_table_ddl = get_table_ddl
class ClickhouseSource(CommonDbSourceService):
@ -142,25 +146,29 @@ class ClickhouseSource(CommonDbSourceService):
return regular_tables + material_tables + view_tables
def get_view_definition(
def get_schema_definition(
self, table_type: str, table_name: str, schema_name: str, inspector: Inspector
) -> Optional[str]:
if table_type in {TableType.View, TableType.MaterializedView}:
definition_fn = inspector.get_view_definition
try:
view_definition = definition_fn(table_name, schema_name)
view_definition = (
"" if view_definition is None else str(view_definition)
"""
Get the DDL statement or View Definition for a table
"""
try:
if table_type in {TableType.View, TableType.MaterializedView}:
definition_fn = inspector.get_view_definition
schema_definition = definition_fn(table_name, schema_name)
else:
schema_definition = inspector.get_table_ddl(
self.connection, table_name, schema_name
)
return view_definition
schema_definition = (
str(schema_definition) if schema_definition is not None else None
)
return schema_definition
except NotImplementedError:
logger.warning("View definition not implemented")
except NotImplementedError:
logger.warning("Schema definition not implemented")
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(
f"Failed to fetch view definition for {table_name}: {exc}"
)
return None
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(f"Failed to fetch schema definition for {table_name}: {exc}")
return None

46
ingestion/src/metadata/ingestion/source/database/common_db_source.py
View File

@ -359,26 +359,32 @@ class CommonDbSourceService(
logger.debug(traceback.format_exc())
@calculate_execution_time()
def get_view_definition(
def get_schema_definition(
self, table_type: str, table_name: str, schema_name: str, inspector: Inspector
) -> Optional[str]:
if table_type in (TableType.View, TableType.MaterializedView):
try:
view_definition = inspector.get_view_definition(table_name, schema_name)
view_definition = (
"" if view_definition is None else str(view_definition)
"""
Get the DDL statement or View Definition for a table
"""
try:
if table_type in (TableType.View, TableType.MaterializedView):
schema_definition = inspector.get_view_definition(
table_name, schema_name
)
return view_definition
except NotImplementedError:
logger.warning("View definition not implemented")
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(
f"Failed to fetch view definition for {table_name}: {exc}"
else:
schema_definition = inspector.get_table_ddl(
self.connection, table_name, schema_name
)
return None
schema_definition = (
str(schema_definition) if schema_definition is not None else None
)
return schema_definition
except NotImplementedError:
logger.warning("Schema definition not implemented")
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(f"Failed to fetch schema definition for {table_name}: {exc}")
return None
def is_partition( # pylint: disable=unused-argument
@ -449,7 +455,7 @@ class CommonDbSourceService(
inspector=self.inspector,
)
view_definition = self.get_view_definition(
schema_definition = self.get_schema_definition(
table_type=table_type,
table_name=table_name,
schema_name=schema_name,
@ -468,7 +474,7 @@ class CommonDbSourceService(
),
columns=columns,
tableConstraints=table_constraints,
viewDefinition=view_definition,
schemaDefinition=schema_definition,
databaseSchema=fqn.build(
metadata=self.metadata,
entity_type=DatabaseSchema,
@ -501,13 +507,13 @@ class CommonDbSourceService(
self.register_record(table_request=table_request)
# Flag view as visited
if table_type == TableType.View or view_definition:
if table_type == TableType.View or schema_definition:
table_view = TableView.parse_obj(
{
"table_name": table_name,
"schema_name": schema_name,
"db_name": self.context.get().database,
"view_definition": view_definition,
"view_definition": schema_definition,
}
)
self.context.get_global().table_views.append(table_view)

30
ingestion/src/metadata/ingestion/source/database/databricks/metadata.py
View File

@ -60,7 +60,9 @@ from metadata.utils.constants import DEFAULT_DATABASE
from metadata.utils.filters import filter_by_database
from metadata.utils.logger import ingestion_logger
from metadata.utils.sqlalchemy_utils import (
get_all_table_ddls,
get_all_view_definitions,
get_table_ddl,
get_view_definition_wrapper,
)
from metadata.utils.tag_utils import get_ometa_tag_and_classification
@ -129,7 +131,6 @@ def get_columns(self, connection, table_name, schema=None, **kw):
value should match what is provided in the 'source.config.database' field in the
Databricks ingest config file.
"""
db_name = kw["db_name"] if "db_name" in kw else None
rows = _get_column_rows(self, connection, table_name, schema)
result = []
@ -157,20 +158,13 @@ def get_columns(self, connection, table_name, schema=None, **kw):
"system_data_type": raw_col_type,
}
if col_type in {"array", "struct", "map"}:
if db_name and schema:
rows = dict(
connection.execute(
f"DESCRIBE {db_name}.{schema}.{table_name} {col_name}"
).fetchall()
)
else:
rows = dict(
connection.execute(
f"DESCRIBE {schema}.{table_name} {col_name}"
if schema
else f"DESCRIBE {table_name} {col_name}"
).fetchall()
)
rows = dict(
connection.execute(
f"DESCRIBE {schema}.{table_name} {col_name}"
if schema
else f"DESCRIBE {table_name} {col_name}"
).fetchall()
)
col_info["system_data_type"] = rows["data_type"]
col_info["is_complex"] = True
@ -259,6 +253,8 @@ DatabricksDialect.get_schema_names = get_schema_names
DatabricksDialect.get_view_definition = get_view_definition
DatabricksDialect.get_all_view_definitions = get_all_view_definitions
reflection.Inspector.get_schema_names = get_schema_names_reflection
reflection.Inspector.get_all_table_ddls = get_all_table_ddls
reflection.Inspector.get_table_ddl = get_table_ddl
class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDBSource):
@ -596,9 +592,7 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB
try:
cursor = self.connection.execute(
DATABRICKS_GET_TABLE_COMMENTS.format(
schema_name=schema_name,
table_name=table_name,
catalog_name=self.context.get().database,
schema_name=schema_name, table_name=table_name
)
)
for result in list(cursor):

5
ingestion/src/metadata/ingestion/source/database/databricks/queries.py
View File

@ -24,10 +24,7 @@ DATABRICKS_VIEW_DEFINITIONS = textwrap.dedent(
"""
)
DATABRICKS_GET_TABLE_COMMENTS = (
"DESCRIBE TABLE EXTENDED {catalog_name}.{schema_name}.{table_name}"
)
DATABRICKS_GET_TABLE_COMMENTS = "DESCRIBE TABLE EXTENDED {schema_name}.{table_name}"
DATABRICKS_GET_CATALOGS = "SHOW CATALOGS"

2
ingestion/src/metadata/ingestion/source/database/deltalake/metadata.py
View File

@ -284,7 +284,7 @@ class DeltalakeSource(DatabaseServiceSource):
database_name=self.context.get().database,
schema_name=schema_name,
),
viewDefinition=view_definition,
schemaDefinition=view_definition,
)
yield Either(right=table_request)

23
ingestion/src/metadata/ingestion/source/database/hive/metadata.py
View File

@ -12,9 +12,11 @@
Hive source methods.
"""
import traceback
from typing import Optional, Tuple
from pyhive.sqlalchemy_hive import HiveDialect
from sqlalchemy.engine.reflection import Inspector
from metadata.generated.schema.entity.services.connections.database.hiveConnection import (
HiveConnection,
@ -96,3 +98,24 @@ class HiveSource(CommonDbSourceService):
)
self._connection_map = {} # Lazy init as well
self._inspector_map = {}
def get_schema_definition( # pylint: disable=unused-argument
self, table_type: str, table_name: str, schema_name: str, inspector: Inspector
) -> Optional[str]:
"""
Get the DDL statement or View Definition for a table
"""
try:
schema_definition = inspector.get_view_definition(table_name, schema_name)
schema_definition = (
str(schema_definition) if schema_definition is not None else None
)
return schema_definition
except NotImplementedError:
logger.warning("Schema definition not implemented")
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(f"Failed to fetch schema definition for {table_name}: {exc}")
return None

6
ingestion/src/metadata/ingestion/source/database/mssql/metadata.py
View File

@ -13,6 +13,7 @@ import traceback
from typing import Dict, Iterable, List, Optional
from sqlalchemy.dialects.mssql.base import MSDialect, ischema_names
from sqlalchemy.engine.reflection import Inspector
from metadata.generated.schema.api.data.createStoredProcedure import (
CreateStoredProcedureRequest,
@ -65,7 +66,9 @@ from metadata.utils.logger import ingestion_logger
from metadata.utils.sqa_utils import update_mssql_ischema_names
from metadata.utils.sqlalchemy_utils import (
get_all_table_comments,
get_all_table_ddls,
get_all_view_definitions,
get_table_ddl,
)
logger = ingestion_logger()
@ -87,6 +90,9 @@ MSDialect.get_foreign_keys = get_foreign_keys
MSDialect.get_table_names = get_table_names
MSDialect.get_view_names = get_view_names
Inspector.get_all_table_ddls = get_all_table_ddls
Inspector.get_table_ddl = get_table_ddl
class MssqlSource(StoredProcedureMixin, CommonDbSourceService, MultiDBSource):
"""

5
ingestion/src/metadata/ingestion/source/database/mysql/metadata.py
View File

@ -13,6 +13,7 @@ from typing import Optional, cast
from sqlalchemy.dialects.mysql.base import ischema_names
from sqlalchemy.dialects.mysql.reflection import MySQLTableDefinitionParser
from sqlalchemy.engine.reflection import Inspector
from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
MysqlConnection,
@ -24,6 +25,7 @@ from metadata.ingestion.api.steps import InvalidSourceException
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
from metadata.ingestion.source.database.mysql.utils import col_type_map, parse_column
from metadata.utils.sqlalchemy_utils import get_all_table_ddls, get_table_ddl
ischema_names.update(col_type_map)
@ -32,6 +34,9 @@ MySQLTableDefinitionParser._parse_column = ( # pylint: disable=protected-access
parse_column
)
Inspector.get_all_table_ddls = get_all_table_ddls
Inspector.get_table_ddl = get_table_ddl
class MysqlSource(CommonDbSourceService):
"""

40
ingestion/src/metadata/ingestion/source/database/oracle/metadata.py
View File

@ -71,7 +71,9 @@ from metadata.utils.helpers import get_start_and_end
from metadata.utils.logger import ingestion_logger
from metadata.utils.sqlalchemy_utils import (
get_all_table_comments,
get_all_table_ddls,
get_all_view_definitions,
get_table_ddl,
)
logger = ingestion_logger()
@ -96,6 +98,9 @@ Inspector.get_mview_names = get_mview_names
Inspector.get_mview_definition = get_mview_definition
OracleDialect.get_mview_names = get_mview_names_dialect
Inspector.get_all_table_ddls = get_all_table_ddls
Inspector.get_table_ddl = get_table_ddl
class OracleSource(StoredProcedureMixin, CommonDbSourceService):
"""
@ -138,27 +143,36 @@ class OracleSource(StoredProcedureMixin, CommonDbSourceService):
return regular_tables + material_tables
def get_view_definition(
def get_schema_definition(
self, table_type: str, table_name: str, schema_name: str, inspector: Inspector
) -> Optional[str]:
if table_type not in {TableType.View, TableType.MaterializedView}:
return None
definition_fn = inspector.get_view_definition
if table_type == TableType.MaterializedView:
definition_fn = inspector.get_mview_definition
"""
Get the DDL statement or View Definition for a table
"""
try:
view_definition = definition_fn(table_name, schema_name)
view_definition = "" if view_definition is None else str(view_definition)
return view_definition
if table_type not in {TableType.View, TableType.MaterializedView}:
schema_definition = inspector.get_table_ddl(
self.connection, table_name, schema_name
)
else:
definition_fn = inspector.get_view_definition
if table_type == TableType.MaterializedView:
definition_fn = inspector.get_mview_definition
schema_definition = definition_fn(table_name, schema_name)
schema_definition = (
str(schema_definition) if schema_definition is not None else None
)
return schema_definition
except NotImplementedError:
logger.warning("View definition not implemented")
logger.warning("Schema definition not implemented")
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(f"Failed to fetch view definition for {table_name}: {exc}")
logger.warning(f"Failed to fetch Schema definition for {table_name}: {exc}")
return None
def process_result(self, data: FetchProcedureList):

4
ingestion/src/metadata/ingestion/source/database/postgres/metadata.py
View File

@ -66,8 +66,10 @@ from metadata.utils.filters import filter_by_database
from metadata.utils.logger import ingestion_logger
from metadata.utils.sqlalchemy_utils import (
get_all_table_comments,
get_all_table_ddls,
get_all_table_owners,
get_all_view_definitions,
get_table_ddl,
)
from metadata.utils.tag_utils import get_ometa_tag_and_classification
@ -124,6 +126,8 @@ PGDialect.get_all_table_owners = get_all_table_owners
PGDialect.get_table_owner = get_table_owner
PGDialect.ischema_names = ischema_names
Inspector.get_all_table_ddls = get_all_table_ddls
Inspector.get_table_ddl = get_table_ddl
Inspector.get_table_owner = get_etable_owner
PGDialect.get_foreign_keys = get_foreign_keys

9
ingestion/src/metadata/ingestion/source/database/redshift/metadata.py
View File

@ -96,7 +96,11 @@ from metadata.utils.execution_time_tracker import (
from metadata.utils.filters import filter_by_database
from metadata.utils.helpers import get_start_and_end
from metadata.utils.logger import ingestion_logger
from metadata.utils.sqlalchemy_utils import get_all_table_comments
from metadata.utils.sqlalchemy_utils import (
get_all_table_comments,
get_all_table_ddls,
get_table_ddl,
)
logger = ingestion_logger()
@ -122,6 +126,9 @@ RedshiftDialect._get_all_relation_info = ( # pylint: disable=protected-access
_get_all_relation_info
)
Inspector.get_all_table_ddls = get_all_table_ddls
Inspector.get_table_ddl = get_table_ddl
class RedshiftSource(
LifeCycleQueryMixin, StoredProcedureMixin, CommonDbSourceService, MultiDBSource

4
ingestion/src/metadata/ingestion/source/database/sample_data.py
View File

@ -733,6 +733,7 @@ class SampleDataSource(
tableType=table["tableType"],
tableConstraints=table.get("tableConstraints"),
tags=table["tags"],
schemaDefinition=table.get("schemaDefinition"),
)
yield Either(right=table_and_db)
@ -1567,9 +1568,6 @@ class SampleDataSource(
def ingest_life_cycle(self) -> Iterable[Either[OMetaLifeCycleData]]:
"""Iterate over all the life cycle data and ingest them"""
for table_life_cycle in self.life_cycle_data["lifeCycleData"]:
table = self.metadata.get_by_name(
entity=Table, fqn=table_life_cycle["fqn"], fields=["lifeCycle"]
)
life_cycle = table_life_cycle["lifeCycle"]
life_cycle_data = LifeCycle()
life_cycle_data.created = AccessDetails(

5
ingestion/src/metadata/ingestion/source/database/snowflake/metadata.py
View File

@ -92,6 +92,7 @@ from metadata.ingestion.source.database.snowflake.utils import (
get_schema_columns,
get_schema_foreign_keys,
get_table_comment,
get_table_ddl,
get_table_names,
get_table_names_reflection,
get_unique_constraints,
@ -108,7 +109,7 @@ from metadata.utils import fqn
from metadata.utils.filters import filter_by_database
from metadata.utils.helpers import get_start_and_end
from metadata.utils.logger import ingestion_logger
from metadata.utils.sqlalchemy_utils import get_all_table_comments
from metadata.utils.sqlalchemy_utils import get_all_table_comments, get_all_table_ddls
from metadata.utils.tag_utils import get_ometa_tag_and_classification
ischema_names["VARIANT"] = VARIANT
@ -137,6 +138,8 @@ SnowflakeDialect._current_database_schema = ( # pylint: disable=protected-acces
SnowflakeDialect.get_pk_constraint = get_pk_constraint
SnowflakeDialect.get_foreign_keys = get_foreign_keys
SnowflakeDialect.get_columns = get_columns
Inspector.get_all_table_ddls = get_all_table_ddls
Inspector.get_table_ddl = get_table_ddl
SnowflakeDialect._get_schema_foreign_keys = get_schema_foreign_keys

4
ingestion/src/metadata/ingestion/source/database/snowflake/queries.py
View File

@ -351,3 +351,7 @@ JOIN Q_HISTORY Q
ORDER BY PROCEDURE_START_TIME DESC
"""
)
SNOWFLAKE_GET_TABLE_DDL = """
SELECT GET_DDL('TABLE','{table_name}') AS \"text\"
"""

20
ingestion/src/metadata/ingestion/source/database/snowflake/utils.py
View File

@ -35,6 +35,7 @@ from metadata.ingestion.source.database.snowflake.queries import (
SNOWFLAKE_GET_EXTERNAL_TABLE_NAMES,
SNOWFLAKE_GET_MVIEW_NAMES,
SNOWFLAKE_GET_SCHEMA_COLUMNS,
SNOWFLAKE_GET_TABLE_DDL,
SNOWFLAKE_GET_TRANSIENT_NAMES,
SNOWFLAKE_GET_VIEW_NAMES,
SNOWFLAKE_GET_WITHOUT_TRANSIENT_TABLE_NAMES,
@ -476,3 +477,22 @@ def get_columns(self, connection, table_name, schema=None, **kw):
if normalized_table_name not in schema_columns:
raise sa_exc.NoSuchTableError()
return schema_columns[normalized_table_name]
@reflection.cache
def get_table_ddl(
self, connection, table_name, schema=None, **kw
): # pylint: disable=unused-argument
"""
Gets the Table DDL
"""
schema = schema or self.default_schema_name
table_name = f"{schema}.{table_name}" if schema else table_name
cursor = connection.execute(SNOWFLAKE_GET_TABLE_DDL.format(table_name=table_name))
try:
result = cursor.fetchone()
if result:
return result[0]
except Exception:
pass
return None

4
ingestion/src/metadata/ingestion/source/database/sqlalchemy_source.py
View File

@ -74,11 +74,11 @@ class SqlAlchemySource(ABC):
"""
@abstractmethod
def get_view_definition(
def get_schema_definition(
self, table_type, table_name: str, schema_name: str, inspector: Inspector
) -> Optional[str]:
"""
Method to fetch view definition
Method to fetch schema definition
"""
@abstractmethod

46
ingestion/src/metadata/utils/sqlalchemy_utils.py
View File

@ -12,10 +12,15 @@
"""
Module for sqlalchemy dialect utils
"""
import traceback
from typing import Dict, Optional, Tuple
from sqlalchemy.engine import Engine, reflection
from sqlalchemy.schema import CreateTable, MetaData
from metadata.utils.logger import ingestion_logger
logger = ingestion_logger()
@reflection.cache
@ -125,3 +130,42 @@ def convert_numpy_to_list(data):
if isinstance(data, dict):
return {key: convert_numpy_to_list(value) for key, value in data.items()}
return data
@reflection.cache
def get_all_table_ddls(
self, connection, query, schema_name, **kw
): # pylint: disable=unused-argument
"""
Method to fetch ddl of all available tables
"""
try:
self.all_table_ddls: Dict[Tuple[str, str], str] = {}
self.current_db: str = schema_name
meta = MetaData()
meta.reflect(bind=connection.engine, schema=schema_name)
for table in meta.sorted_tables or []:
self.all_table_ddls[(table.schema, table.name)] = str(CreateTable(table))
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(f"Failed to get table ddls for {schema_name}: {exc}")
def get_table_ddl_wrapper(
self, connection, query, table_name, schema=None, **kw
): # pylint: disable=unused-argument
if not hasattr(self, "all_table_ddls") or self.current_db != schema:
self.get_all_table_ddls(connection, query, schema)
return self.all_table_ddls.get((schema, table_name), "")
def get_table_ddl(
self, connection, table_name, schema=None, **kw
): # pylint: disable=unused-argument
return get_table_ddl_wrapper(
self,
connection=connection,
query=None,
table_name=table_name,
schema=schema,
)

12
ingestion/tests/unit/topology/database/test_bigquery.py
View File

@ -177,7 +177,7 @@ MOCK_TABLE = Table(
id="c3eb265f-5445-4ad3-ba5e-797d3a3071bb", type="databaseSchema"
),
tags=[],
viewDefinition=None,
schemaDefinition=None,
retentionPeriod=None,
extension=None,
sourceUrl=SourceUrl(
@ -411,7 +411,7 @@ EXPECTED_TABLE = [
__root__="bigquery_source_test.random-project-id.sample_schema"
),
tags=[],
viewDefinition=None,
schemaDefinition=None,
retentionPeriod=None,
extension=None,
sourceUrl=SourceUrl(
@ -507,7 +507,7 @@ EXPECTED_TABLE = [
__root__="bigquery_source_test.random-project-id.sample_schema"
),
tags=[],
viewDefinition=None,
schemaDefinition=None,
retentionPeriod=None,
extension=None,
sourceUrl=SourceUrl(
@ -652,6 +652,12 @@ class BigqueryUnitTest(TestCase):
i
] # pylint: disable=cell-var-from-loop
)
self.bq_source.inspector.get_table_ddl = (
lambda table_name, schema, db_name: None # pylint: disable=cell-var-from-loop
)
self.bq_source.inspector.get_table_comment = (
lambda table_name, schema, db_name: None # pylint: disable=cell-var-from-loop
)
assert EXPECTED_TABLE[i] == [
either.right for either in self.bq_source.yield_table(table)
]

2
ingestion/tests/unit/topology/database/test_databricks.py
View File

@ -240,7 +240,7 @@ EXPTECTED_TABLE = [
__root__="local_databricks.hive_metastore.do_it_all_with_default_schema"
),
tags=None,
viewDefinition=None,
schemaDefinition=None,
extension=None,
)
]

2
ingestion/tests/unit/topology/database/test_deltalake.py
View File

@ -220,7 +220,7 @@ class DeltaLakeUnitTest(TestCase):
columns=expected_columns,
tableConstraints=None,
databaseSchema=MOCK_DATABASE_SCHEMA.fullyQualifiedName,
viewDefinition=None,
schemaDefinition=None,
)
self.assertEqual(table_request, expected_table_request)

2
ingestion/tests/unit/topology/database/test_domodatabase.py
View File

@ -230,7 +230,7 @@ EXPTECTED_TABLE = [
owner=None,
databaseSchema="domodashboard_source_test.do_it_all_with_default_config.do_it_all_with_default_schema",
tags=None,
viewDefinition=None,
schemaDefinition=None,
extension=None,
)
]

2
ingestion/tests/unit/topology/database/test_hive.py
View File

@ -272,7 +272,7 @@ EXPECTED_TABLE = [
__root__="hive_source_test.sample_database.sample_schema"
),
tags=None,
viewDefinition=None,
schemaDefinition=None,
retentionPeriod=None,
extension=None,
sourceUrl=None,

2
ingestion/tests/unit/topology/database/test_mssql.py
View File

@ -274,7 +274,7 @@ EXPECTED_TABLE = [
__root__='mssql_source_test.sample_database."sample.schema"'
),
tags=None,
viewDefinition=None,
schemaDefinition=None,
retentionPeriod=None,
extension=None,
sourceUrl=None,

2
ingestion/tests/unit/topology/database/test_sas.py
View File

@ -236,7 +236,7 @@ EXPECTED_TABLE = Table(
),
serviceType="SAS",
location=None,
viewDefinition=None,
schemaDefinition=None,
usageSummary=None,
followers=None,
joins=None,

1
ingestion/tests/unit/topology/metadata/test_amundsen.py
View File

@ -82,6 +82,7 @@ EXPECTED_SERVICE = [
connectionArguments=None,
supportsMetadataExtraction=True,
supportsProfiler=True,
supportsDDL=True,
supportsDBTExtraction=True,
)
),

2
ingestion/tests/unit/topology/metadata/test_atlas.py
View File

@ -276,7 +276,7 @@ EXPTECTED_TABLE = Table(
),
serviceType="Hive",
location=None,
viewDefinition=None,
schemaDefinition=None,
tags=[
TagLabel(
tagFQN="AtlasMetadata.atlas_table",

2
openmetadata-docs/content/v1.1.x/main-concepts/metadata-standard/schemas/api/data/createTable.md
View File

@ -23,7 +23,7 @@ slug: /main-concepts/metadata-standard/schemas/api/data/createtable
- **`databaseSchema`**: FullyQualified name of the Schema corresponding to this table. Refer to *../../type/basic.json#/definitions/fullyQualifiedEntityName*.
- **`tags`** *(array)*: Tags for this table. Default: `None`.
- **Items**: Refer to *../../type/tagLabel.json*.
- **`viewDefinition`**: View Definition in SQL. Applies to TableType.View only. Refer to *../../type/basic.json#/definitions/sqlQuery*. Default: `None`.
- **`schemaDefinition`**: Schema Definition or DDL statement of a table in SQL. Refer to *../../type/basic.json#/definitions/sqlQuery*. Default: `None`.
- **`retentionPeriod`**: Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`. Refer to *../../type/basic.json#/definitions/duration*.
- **`extension`**: Entity extension data with custom attributes added to the entity. Refer to *../../type/basic.json#/definitions/entityExtension*.
- **`sourceUrl`**: Source URL of table. Refer to *../../type/basic.json#/definitions/sourceUrl*.

2
openmetadata-docs/content/v1.1.x/main-concepts/metadata-standard/schemas/entity/data/table.md
View File

@ -30,7 +30,7 @@ slug: /main-concepts/metadata-standard/schemas/entity/data/table
- **`service`**: Link to Database service this table is hosted in. Refer to *../../type/entityReference.json*.
- **`serviceType`**: Service type this table is hosted in. Refer to *../services/databaseService.json#/definitions/databaseServiceType*.
- **`location`**: Reference to the Location that contains this table. Refer to *../../type/entityReference.json*.
- **`viewDefinition`**: View Definition in SQL. Applies to TableType.View only. Refer to *../../type/basic.json#/definitions/sqlQuery*.
- **`schemaDefinition`**: Schema Definition or DDL statement of a table in SQL. Refer to *../../type/basic.json#/definitions/sqlQuery*.
- **`tags`** *(array)*: Tags for this table. Default: `None`.
- **Items**: Refer to *../../type/tagLabel.json*.
- **`usageSummary`**: Latest usage information for this table. Refer to *../../type/usageDetails.json*. Default: `None`.

2
openmetadata-docs/content/v1.2.x/main-concepts/metadata-standard/schemas/api/data/createTable.md
View File

@ -23,7 +23,7 @@ slug: /main-concepts/metadata-standard/schemas/api/data/createtable
- **`databaseSchema`**: FullyQualified name of the Schema corresponding to this table. Refer to *../../type/basic.json#/definitions/fullyQualifiedEntityName*.
- **`tags`** *(array)*: Tags for this table. Default: `None`.
- **Items**: Refer to *../../type/tagLabel.json*.
- **`viewDefinition`**: View Definition in SQL. Applies to TableType.View only. Refer to *../../type/basic.json#/definitions/sqlQuery*. Default: `None`.
- **`schemaDefinition`**: Schema Definition or DDL statement of a table in SQL. Refer to *../../type/basic.json#/definitions/sqlQuery*. Default: `None`.
- **`retentionPeriod`**: Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`. Refer to *../../type/basic.json#/definitions/duration*.
- **`extension`**: Entity extension data with custom attributes added to the entity. Refer to *../../type/basic.json#/definitions/entityExtension*.
- **`sourceUrl`**: Source URL of table. Refer to *../../type/basic.json#/definitions/sourceUrl*.

2
openmetadata-docs/content/v1.2.x/main-concepts/metadata-standard/schemas/entity/data/table.md
View File

@ -30,7 +30,7 @@ slug: /main-concepts/metadata-standard/schemas/entity/data/table
- **`service`**: Link to Database service this table is hosted in. Refer to *../../type/entityReference.json*.
- **`serviceType`**: Service type this table is hosted in. Refer to *../services/databaseService.json#/definitions/databaseServiceType*.
- **`location`**: Reference to the Location that contains this table. Refer to *../../type/entityReference.json*.
- **`viewDefinition`**: View Definition in SQL. Applies to TableType.View only. Refer to *../../type/basic.json#/definitions/sqlQuery*.
- **`schemaDefinition`**: Schema Definition or DDL statement of a table in SQL. Refer to *../../type/basic.json#/definitions/sqlQuery*.
- **`tags`** *(array)*: Tags for this table. Default: `None`.
- **Items**: Refer to *../../type/tagLabel.json*.
- **`usageSummary`**: Latest usage information for this table. Refer to *../../type/usageDetails.json*. Default: `None`.

2
openmetadata-docs/content/v1.3.x/main-concepts/metadata-standard/schemas/api/data/createTable.md
View File

@ -23,7 +23,7 @@ slug: /main-concepts/metadata-standard/schemas/api/data/createtable
- **`databaseSchema`**: FullyQualified name of the Schema corresponding to this table. Refer to *../../type/basic.json#/definitions/fullyQualifiedEntityName*.
- **`tags`** *(array)*: Tags for this table. Default: `None`.
- **Items**: Refer to *../../type/tagLabel.json*.
- **`viewDefinition`**: View Definition in SQL. Applies to TableType.View only. Refer to *../../type/basic.json#/definitions/sqlQuery*. Default: `None`.
- **`schemaDefinition`**: Schema Definition or DDL statement of a table in SQL. Refer to *../../type/basic.json#/definitions/sqlQuery*. Default: `None`.
- **`retentionPeriod`**: Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`. Refer to *../../type/basic.json#/definitions/duration*.
- **`extension`**: Entity extension data with custom attributes added to the entity. Refer to *../../type/basic.json#/definitions/entityExtension*.
- **`sourceUrl`**: Source URL of table. Refer to *../../type/basic.json#/definitions/sourceUrl*.

2
openmetadata-docs/content/v1.3.x/main-concepts/metadata-standard/schemas/entity/data/table.md
View File

@ -30,7 +30,7 @@ slug: /main-concepts/metadata-standard/schemas/entity/data/table
- **`service`**: Link to Database service this table is hosted in. Refer to *../../type/entityReference.json*.
- **`serviceType`**: Service type this table is hosted in. Refer to *../services/databaseService.json#/definitions/databaseServiceType*.
- **`location`**: Reference to the Location that contains this table. Refer to *../../type/entityReference.json*.
- **`viewDefinition`**: View Definition in SQL. Applies to TableType.View only. Refer to *../../type/basic.json#/definitions/sqlQuery*.
- **`schemaDefinition`**: Schema Definition or DDL statement of a table in SQL. Refer to *../../type/basic.json#/definitions/sqlQuery*.
- **`tags`** *(array)*: Tags for this table. Default: `None`.
- **Items**: Refer to *../../type/tagLabel.json*.
- **`usageSummary`**: Latest usage information for this table. Refer to *../../type/usageDetails.json*. Default: `None`.

2
openmetadata-docs/content/v1.4.x-SNAPSHOT/main-concepts/metadata-standard/schemas/api/data/createTable.md
View File

@ -23,7 +23,7 @@ slug: /main-concepts/metadata-standard/schemas/api/data/createtable
- **`databaseSchema`**: FullyQualified name of the Schema corresponding to this table. Refer to *../../type/basic.json#/definitions/fullyQualifiedEntityName*.
- **`tags`** *(array)*: Tags for this table. Default: `None`.
- **Items**: Refer to *../../type/tagLabel.json*.
- **`viewDefinition`**: View Definition in SQL. Applies to TableType.View only. Refer to *../../type/basic.json#/definitions/sqlQuery*. Default: `None`.
- **`schemaDefinition`**: Schema Definition or DDL statement of a table in SQL. Refer to *../../type/basic.json#/definitions/sqlQuery*. Default: `None`.
- **`retentionPeriod`**: Retention period of the data in the database. Period is expressed as duration in ISO 8601 format in UTC. Example - `P23DT23H`. Refer to *../../type/basic.json#/definitions/duration*.
- **`extension`**: Entity extension data with custom attributes added to the entity. Refer to *../../type/basic.json#/definitions/entityExtension*.
- **`sourceUrl`**: Source URL of table. Refer to *../../type/basic.json#/definitions/sourceUrl*.

2
openmetadata-docs/content/v1.4.x-SNAPSHOT/main-concepts/metadata-standard/schemas/entity/data/table.md
View File

@ -30,7 +30,7 @@ slug: /main-concepts/metadata-standard/schemas/entity/data/table
- **`service`**: Link to Database service this table is hosted in. Refer to *../../type/entityReference.json*.
- **`serviceType`**: Service type this table is hosted in. Refer to *../services/databaseService.json#/definitions/databaseServiceType*.
- **`location`**: Reference to the Location that contains this table. Refer to *../../type/entityReference.json*.
- **`viewDefinition`**: View Definition in SQL. Applies to TableType.View only. Refer to *../../type/basic.json#/definitions/sqlQuery*.
- **`schemaDefinition`**: Schema Definition or DDL statement of a table in SQL. Refer to *../../type/basic.json#/definitions/sqlQuery*.
- **`tags`** *(array)*: Tags for this table. Default: `None`.
- **Items**: Refer to *../../type/tagLabel.json*.
- **`usageSummary`**: Latest usage information for this table. Refer to *../../type/usageDetails.json*. Default: `None`.

3
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java
View File

@ -180,7 +180,8 @@ public class TableRepository extends EntityRepository<Table> {
fields.contains("tableConstraints") ? table.getTableConstraints() : null);
table.setUsageSummary(fields.contains("usageSummary") ? table.getUsageSummary() : null);
table.setJoins(fields.contains("joins") ? table.getJoins() : null);
table.setViewDefinition(fields.contains("viewDefinition") ? table.getViewDefinition() : null);
table.setSchemaDefinition(
fields.contains("schemaDefinition") ? table.getSchemaDefinition() : null);
table.setTableProfilerConfig(
fields.contains(TABLE_PROFILER_CONFIG) ? table.getTableProfilerConfig() : null);
table.setTestSuite(fields.contains("testSuite") ? table.getTestSuite() : null);

12
openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseUtil.java
View File

@ -24,7 +24,6 @@ import org.openmetadata.schema.type.ColumnDataType;
import org.openmetadata.schema.type.PartitionColumnDetails;
import org.openmetadata.schema.type.TableConstraint;
import org.openmetadata.schema.type.TablePartition;
import org.openmetadata.schema.type.TableType;
public final class DatabaseUtil {
private DatabaseUtil() {}
@ -85,17 +84,6 @@ public final class DatabaseUtil {
}
}
public static void validateViewDefinition(TableType tableType, String viewDefinition) {
if ((tableType == null
|| tableType.equals(TableType.Regular)
|| tableType.equals(TableType.External))
&& viewDefinition != null
&& !viewDefinition.isEmpty()) {
throw new IllegalArgumentException(
"ViewDefinition can only be set on TableType View, SecureView or MaterializedView");
}
}
public static void validateColumns(List<Column> columns) {
validateColumnNames(columns);
for (Column c : columns) {

7
openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java
View File

@ -90,7 +90,7 @@ public class TableResource extends EntityResource<Table, TableRepository> {
public static final String COLLECTION_PATH = "v1/tables/";
static final String FIELDS =
"tableConstraints,tablePartition,usageSummary,owner,customMetrics,columns,"
+ "tags,followers,joins,viewDefinition,dataModel,extension,testSuite,domain,dataProducts,lifeCycle,sourceHash";
+ "tags,followers,joins,schemaDefinition,dataModel,extension,testSuite,domain,dataProducts,lifeCycle,sourceHash";
@Override
public Table addHref(UriInfo uriInfo, Table table) {
@ -109,7 +109,7 @@ public class TableResource extends EntityResource<Table, TableRepository> {
protected List<MetadataOperation> getEntitySpecificOperations() {
allowedFields.add("customMetrics");
addViewOperation(
"columns,tableConstraints,tablePartition,joins,viewDefinition,dataModel",
"columns,tableConstraints,tablePartition,joins,schemaDefinition,dataModel",
MetadataOperation.VIEW_BASIC);
addViewOperation("usageSummary", MetadataOperation.VIEW_USAGE);
addViewOperation("customMetrics", MetadataOperation.VIEW_TESTS);
@ -1215,7 +1215,6 @@ public class TableResource extends EntityResource<Table, TableRepository> {
table.setId(UUID.randomUUID());
DatabaseUtil.validateConstraints(table.getColumns(), table.getTableConstraints());
DatabaseUtil.validateTablePartition(table.getColumns(), table.getTablePartition());
DatabaseUtil.validateViewDefinition(table.getTableType(), table.getViewDefinition());
DatabaseUtil.validateColumns(table.getColumns());
return table;
}
@ -1230,7 +1229,7 @@ public class TableResource extends EntityResource<Table, TableRepository> {
.withTablePartition(create.getTablePartition())
.withTableType(create.getTableType())
.withFileFormat(create.getFileFormat())
.withViewDefinition(create.getViewDefinition())
.withSchemaDefinition(create.getSchemaDefinition())
.withTableProfilerConfig(create.getTableProfilerConfig())
.withDatabaseSchema(
getEntityReference(Entity.DATABASE_SCHEMA, create.getDatabaseSchema())))

3
openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java
View File

@ -21,7 +21,8 @@ public record TableIndex(Table table) implements ColumnIndex {
"sampleData",
"tableProfile",
"joins",
"viewDefinition, tableProfilerConfig, profile, location, tableQueries, tests, dataModel",
"changeDescription",
"schemaDefinition, tableProfilerConfig, profile, location, tableQueries, tests, dataModel",
"testSuite.changeDescription");
@Override

38
openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java
View File

@ -1120,7 +1120,7 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
}
@Test
void put_viewDefinition_200(TestInfo test) throws IOException {
void put_schemaDefinition_200(TestInfo test) throws IOException {
CreateTable createTable = createRequest(test);
createTable.setTableType(TableType.View);
String query =
@ -1132,31 +1132,11 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
select * from spectrum.sales
with no schema binding;
""";
createTable.setViewDefinition(query);
createTable.setSchemaDefinition(query);
Table table = createAndCheckEntity(createTable, ADMIN_AUTH_HEADERS);
table = getEntity(table.getId(), "viewDefinition", ADMIN_AUTH_HEADERS);
LOG.info("table view definition {}", table.getViewDefinition());
assertEquals(table.getViewDefinition(), query);
}
@Test
void put_viewDefinition_invalid_table_4xx(TestInfo test) {
CreateTable createTable = createRequest(test);
createTable.setTableType(TableType.Regular);
String query =
"""
sales_vw
create view sales_vw as
select * from public.sales
union all
select * from spectrum.sales
with no schema binding;
""";
createTable.setViewDefinition(query);
assertResponseContains(
() -> createAndCheckEntity(createTable, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"ViewDefinition can only be set on TableType View, SecureView or MaterializedView");
table = getEntity(table.getId(), "schemaDefinition", ADMIN_AUTH_HEADERS);
LOG.info("table view definition {}", table.getSchemaDefinition());
assertEquals(table.getSchemaDefinition(), query);
}
@Test
@ -2719,25 +2699,25 @@ public class TableResourceTest extends EntityResourceTest<Table, CreateTable> {
table.getFollowers(),
table.getJoins(),
table.getSampleData(),
table.getViewDefinition(),
table.getSchemaDefinition(),
table.getProfile(),
table.getLocation(),
table.getDataModel());
String fields =
"tableConstraints,usageSummary,owner,"
+ "tags,followers,joins,sampleData,viewDefinition,profile,location,dataModel";
+ "tags,followers,joins,sampleData,schemaDefinition,profile,location,dataModel";
table =
byName
? getEntityByName(table.getFullyQualifiedName(), fields, ADMIN_AUTH_HEADERS)
: getEntity(table.getId(), fields, ADMIN_AUTH_HEADERS);
assertListNotNull(table.getService(), table.getServiceType(), table.getColumns());
// Fields sampleData, viewDefinition, tableProfile, location,
// Fields sampleData, schemaDefinition, tableProfile, location,
// and dataModel are not set during creation - tested elsewhere
assertListNotNull(
table.getTableConstraints(),
table.getUsageSummary(),
table.getJoins() /*, table.getSampleData(), table.getViewDefinition(), table
table.getJoins() /*, table.getSampleData(), table.getSchemaDefinition(), table
.getTableProfile(), table.getLocation(), table.getDataModel()*/);
assertListNotEmpty(table.getTableConstraints());
// Checks for other owner, tags, and followers is done in the base class

7
openmetadata-spec/src/main/resources/json/schema/api/data/createTable.json
View File

@ -31,6 +31,9 @@
},
"default": null
},
"dataModel": {
"$ref": "../../entity/data/table.json#/definitions/dataModel"
},
"tableConstraints": {
"type": "array",
"items": {
@ -61,8 +64,8 @@
},
"default": null
},
"viewDefinition": {
"description": "View Definition in SQL. Applies to TableType.View only",
"schemaDefinition": {
"description": "DDL for Tables and Views",
"$ref": "../../type/basic.json#/definitions/sqlQuery",
"default": null
},

14
openmetadata-spec/src/main/resources/json/schema/entity/data/table.json
View File

@ -818,18 +818,22 @@
"additionalProperties": false
},
"modelType": {
"$comment": "Currently only DBT model type is supported",
"$comment": "Currently only DBT and DDL model type is supported",
"enum": [
"DBT"
"DBT",
"DDL"
],
"javaEnums": [
{
"name": "DBT"
},
{
"name": "DDL"
}
]
},
"dataModel": {
"description": "This captures information about how the table is modeled. Currently only DBT model is supported.",
"description": "This captures information about how the table is modeled. Currently only DBT and DDL model is supported.",
"type": "object",
"javaType": "org.openmetadata.schema.type.DataModel",
"properties": {
@ -985,8 +989,8 @@
"description": "Reference to the Location that contains this table.",
"$ref": "../../type/entityReference.json"
},
"viewDefinition": {
"description": "View Definition in SQL. Applies to TableType.View only.",
"schemaDefinition": {
"description": "DDL for Tables and Views",
"$ref": "../../type/basic.json#/definitions/sqlQuery"
},
"tags": {

5
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/connectionBasicType.json
View File

@ -40,6 +40,11 @@
"type": "boolean",
"default": true
},
"supportsDDL": {
"description": "Supports DDL",
"type": "boolean",
"default": true
},
"supportsDatabase": {
"description": "The source service supports the database concept in its hierarchy",
"type": "boolean",

4
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json
View File

@ -93,6 +93,10 @@
"title": "Supports Profiler",
"$ref": "../connectionBasicType.json#/definitions/supportsProfiler"
},
"supportsDDL": {
"title": "Supports DDL",
"$ref": "../connectionBasicType.json#/definitions/supportsDDL"
},
"supportsDatabase": {
"title": "Supports Database",
"$ref": "../connectionBasicType.json#/definitions/supportsDatabase"

4
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/clickhouseConnection.json
View File

@ -103,6 +103,10 @@
"title": "Supports Profiler",
"$ref": "../connectionBasicType.json#/definitions/supportsProfiler"
},
"supportsDDL": {
"title": "Supports DDL",
"$ref": "../connectionBasicType.json#/definitions/supportsDDL"
},
"supportsQueryComment": {
"title": "Supports Query Comment",
"$ref": "../connectionBasicType.json#/definitions/supportsQueryComment"

4
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/databricksConnection.json
View File

@ -89,6 +89,10 @@
"title": "Supports Profiler",
"$ref": "../connectionBasicType.json#/definitions/supportsProfiler"
},
"supportsDDL": {
"title": "Supports DDL",
"$ref": "../connectionBasicType.json#/definitions/supportsDDL"
},
"supportsDatabase": {
"title": "Supports Database",
"$ref": "../connectionBasicType.json#/definitions/supportsDatabase"

4
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/hiveConnection.json
View File

@ -111,6 +111,10 @@
"title": "Supports Profiler",
"$ref": "../connectionBasicType.json#/definitions/supportsProfiler"
},
"supportsDDL": {
"title": "Supports DDL",
"$ref": "../connectionBasicType.json#/definitions/supportsDDL"
},
"sampleDataStorageConfig": {
"title": "Storage Config for Sample Data",
"$ref": "../connectionBasicType.json#/definitions/sampleDataStorageConfig"

4
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json
View File

@ -84,6 +84,10 @@
"title": "Supports Profiler",
"$ref": "../connectionBasicType.json#/definitions/supportsProfiler"
},
"supportsDDL": {
"title": "Supports DDL",
"$ref": "../connectionBasicType.json#/definitions/supportsDDL"
},
"supportsDatabase": {
"title": "Supports Database",
"$ref": "../connectionBasicType.json#/definitions/supportsDatabase"

4
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json
View File

@ -99,6 +99,10 @@
"title": "Supports Query Comment",
"$ref": "../connectionBasicType.json#/definitions/supportsQueryComment"
},
"supportsDDL": {
"title": "Supports DDL",
"$ref": "../connectionBasicType.json#/definitions/supportsDDL"
},
"sampleDataStorageConfig": {
"title": "Storage Config for Sample Data",
"$ref": "../connectionBasicType.json#/definitions/sampleDataStorageConfig"

4
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/oracleConnection.json
View File

@ -136,6 +136,10 @@
"title": "Supports Profiler",
"$ref": "../connectionBasicType.json#/definitions/supportsProfiler"
},
"supportsDDL": {
"title": "Supports DDL",
"$ref": "../connectionBasicType.json#/definitions/supportsDDL"
},
"supportsQueryComment": {
"title": "Supports Query Comment",
"$ref": "../connectionBasicType.json#/definitions/supportsQueryComment"

4
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json
View File

@ -110,6 +110,10 @@
"title": "Supports Profiler",
"$ref": "../connectionBasicType.json#/definitions/supportsProfiler"
},
"supportsDDL": {
"title": "Supports DDL",
"$ref": "../connectionBasicType.json#/definitions/supportsDDL"
},
"supportsDatabase": {
"title": "Supports Database",
"$ref": "../connectionBasicType.json#/definitions/supportsDatabase"

4
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/redshiftConnection.json
View File

@ -94,6 +94,10 @@
"title": "Supports Profiler",
"$ref": "../connectionBasicType.json#/definitions/supportsProfiler"
},
"supportsDDL": {
"title": "Supports DDL",
"$ref": "../connectionBasicType.json#/definitions/supportsDDL"
},
"supportsDatabase": {
"title": "Supports Database",
"$ref": "../connectionBasicType.json#/definitions/supportsDatabase"

4
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json
View File

@ -117,6 +117,10 @@
"title": "Supports Profiler",
"$ref": "../connectionBasicType.json#/definitions/supportsProfiler"
},
"supportsDDL": {
"title": "Supports DDL",
"$ref": "../connectionBasicType.json#/definitions/supportsDDL"
},
"supportsDatabase": {
"title": "Supports Database",
"$ref": "../connectionBasicType.json#/definitions/supportsDatabase"

6
openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json
View File

@ -90,6 +90,12 @@
"default": true,
"title": "Include Stored Procedures"
},
"includeDDL": {
"description": "Optional configuration to toggle the DDL Statements ingestion.",
"type": "boolean",
"default": true,
"title": "Include DDL Statements"
},
"queryLogDuration": {
"description": "Configuration to tune how far we want to look back in query logs to process Stored Procedures results.",
"type": "integer",

1
openmetadata-ui/src/main/resources/ui/src/components/common/QueryViewer/QueryViewer.component.tsx
View File

@ -69,6 +69,7 @@ const QueryViewer = ({
)}
mode={{ name: CSMode.SQL }}
options={{ readOnly: true }}
showCopyButton={false}
value={sqlQuery}
/>
</Card>

2
openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts
View File

@ -115,6 +115,7 @@ export enum TabSpecificField {
SCHEDULE_INTERVAL = 'scheduleInterval',
TESTSUITE = 'testSuite',
VIEW_DEFINITION = 'viewDefinition',
SCHEMA_DEFINITION = 'schemaDefinition',
FIELDS = 'fields',
VOTES = 'votes',
DOMAIN = 'domain',
@ -141,6 +142,7 @@ export enum EntityTabs {
LINEAGE = 'lineage',
DBT = 'dbt',
VIEW_DEFINITION = 'view_definition',
SCHEMA_DEFINITION = 'schema_definition',
CUSTOM_PROPERTIES = 'custom_properties',
MODEL = 'model',
FEATURES = 'features',

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json
View File

@ -969,6 +969,7 @@
"schedule-to-run-every": "Zeitplan für die Ausführung alle",
"schedule-type": "Schedule Type",
"schema": "Schema",
"schema-definition": "Schema Definition",
"schema-field": "Schemafeld",
"schema-field-plural": "Schemafelder",
"schema-name": "Schema-Name",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json
View File

@ -969,6 +969,7 @@
"schedule-to-run-every": "Scheduled to run every",
"schedule-type": "Schedule Type",
"schema": "Schema",
"schema-definition": "Schema Definition",
"schema-field": "Schema Field",
"schema-field-plural": "Schema Fields",
"schema-name": "Schema Name",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json
View File

@ -969,6 +969,7 @@
"schedule-to-run-every": "Programado para ejecutarse cada",
"schedule-type": "Tipo de programación",
"schema": "Esquema",
"schema-definition": "Schema Definition",
"schema-field": "Campo de Esquema",
"schema-field-plural": "Campos de Esquema",
"schema-name": "Nombre del Esquema",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json
View File

@ -969,6 +969,7 @@
"schedule-to-run-every": "Programmer l'exécution toutes les",
"schedule-type": "Schedule Type",
"schema": "Schéma",
"schema-definition": "Schema Definition",
"schema-field": "Champ de Schéma",
"schema-field-plural": "Champs de Schéma",
"schema-name": "Nom du Schéma",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json
View File

@ -969,6 +969,7 @@
"schedule-to-run-every": "מתוזמן להפעלה כל",
"schedule-type": "סוג לוח זמנים",
"schema": "סכימה",
"schema-definition": "Schema Definition",
"schema-field": "שדה סכימה",
"schema-field-plural": "שדות סכימה",
"schema-name": "שם סכימה",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json
View File

@ -969,6 +969,7 @@
"schedule-to-run-every": "Scheduled to run every",
"schedule-type": "Schedule Type",
"schema": "スキーマ",
"schema-definition": "Schema Definition",
"schema-field": "スキーマのフィールド",
"schema-field-plural": "Schema Fields",
"schema-name": "スキーマ名",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json
View File

@ -969,6 +969,7 @@
"schedule-to-run-every": "Gepland voor uitvoering elke",
"schedule-type": "Schematype",
"schema": "Schema",
"schema-definition": "Schema Definition",
"schema-field": "Schemaveld",
"schema-field-plural": "Schemavelden",
"schema-name": "Schemanaam",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json
View File

@ -969,6 +969,7 @@
"schedule-to-run-every": "Agendado para executar a cada",
"schedule-type": "Tipo de Agendamento",
"schema": "Esquema",
"schema-definition": "Schema Definition",
"schema-field": "Campo do Esquema",
"schema-field-plural": "Campos do Esquema",
"schema-name": "Nome do Esquema",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json
View File

@ -969,6 +969,7 @@
"schedule-to-run-every": "Планируется запуск каждые",
"schedule-type": "Schedule Type",
"schema": "Схема",
"schema-definition": "Schema Definition",
"schema-field": "Поле схемы",
"schema-field-plural": "Поля схемы",
"schema-name": "Наименование схемы",

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json
View File

@ -969,6 +969,7 @@
"schedule-to-run-every": "运行时间被设置为每",
"schedule-type": "Schedule Type",
"schema": "Schema",
"schema-definition": "Schema Definition",
"schema-field": "Schema 字段",
"schema-field-plural": "Schema 字段",
"schema-name": "Schema 名称",

34
openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.test.tsx
View File

@ -13,6 +13,7 @@
import { act, render, screen } from '@testing-library/react';
import React from 'react';
import { usePermissionProvider } from '../../context/PermissionProvider/PermissionProvider';
import { TableType } from '../../generated/entity/data/table';
import { getTableDetailsByFQN } from '../../rest/tableAPI';
import { DEFAULT_ENTITY_PERMISSION } from '../../utils/PermissionsUtils';
import TableDetailsPageV1 from './TableDetailsPageV1';
@ -22,7 +23,7 @@ const mockEntityPermissionByFqn = jest
.mockImplementation(() => DEFAULT_ENTITY_PERMISSION);
const COMMON_API_FIELDS =
'columns,followers,joins,tags,owner,dataModel,tableConstraints,viewDefinition,domain,dataProducts,votes,extension';
'columns,followers,joins,tags,owner,dataModel,tableConstraints,schemaDefinition,domain,dataProducts,votes,extension';
jest.mock('../../context/PermissionProvider/PermissionProvider', () => ({
usePermissionProvider: jest.fn().mockImplementation(() => ({
@ -304,7 +305,7 @@ describe('TestDetailsPageV1 component', () => {
expect(screen.queryByText('label.view-definition')).not.toBeInTheDocument();
});
it('TableDetailsPageV1 should render view defination if data is present', async () => {
it('TableDetailsPageV1 should render schema definition tab table type is not view', async () => {
(usePermissionProvider as jest.Mock).mockImplementationOnce(() => ({
getEntityPermissionByFqn: jest.fn().mockImplementationOnce(() => ({
ViewBasic: true,
@ -315,7 +316,7 @@ describe('TestDetailsPageV1 component', () => {
Promise.resolve({
name: 'test',
id: '123',
viewDefinition: 'viewDefinition',
schemaDefinition: 'schemaDefinition query',
})
);
@ -323,11 +324,30 @@ describe('TestDetailsPageV1 component', () => {
render(<TableDetailsPageV1 />);
});
expect(screen.queryByText('label.dbt-lowercase')).not.toBeInTheDocument();
expect(screen.getByText('label.schema-definition')).toBeInTheDocument();
});
expect(
await screen.findByText('label.view-definition')
).toBeInTheDocument();
it('TableDetailsPageV1 should render view definition tab if table type is view', async () => {
(usePermissionProvider as jest.Mock).mockImplementationOnce(() => ({
getEntityPermissionByFqn: jest.fn().mockImplementationOnce(() => ({
ViewBasic: true,
})),
}));
(getTableDetailsByFQN as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({
name: 'test',
id: '123',
schemaDefinition: 'viewDefinition query',
tableType: TableType.View,
})
);
await act(async () => {
render(<TableDetailsPageV1 />);
});
expect(screen.getByText('label.view-definition')).toBeInTheDocument();
});
it('TableDetailsPageV1 should render schemaTab by default', async () => {

33
openmetadata-ui/src/main/resources/ui/src/pages/TableDetailsPageV1/TableDetailsPageV1.tsx
View File

@ -64,7 +64,11 @@ import {
} from '../../enums/entity.enum';
import { CreateThread } from '../../generated/api/feed/createThread';
import { Tag } from '../../generated/entity/classification/tag';
import { JoinedWith, Table } from '../../generated/entity/data/table';
import {
JoinedWith,
Table,
TableType,
} from '../../generated/entity/data/table';
import { Suggestion } from '../../generated/entity/feed/suggestion';
import { ThreadType } from '../../generated/entity/feed/thread';
import { TagLabel } from '../../generated/type/tagLabel';
@ -152,6 +156,11 @@ const TableDetailsPageV1: React.FC = () => {
[datasetFQN]
);
const isViewTableType = useMemo(
() => tableDetails?.tableType === TableType.View,
[tableDetails?.tableType]
);
const fetchTableDetails = useCallback(async () => {
setLoading(true);
try {
@ -730,13 +739,25 @@ const TableDetailsPageV1: React.FC = () => {
{
label: (
<TabsLabel
id={EntityTabs.VIEW_DEFINITION}
name={t('label.view-definition')}
id={
isViewTableType
? EntityTabs.VIEW_DEFINITION
: EntityTabs.SCHEMA_DEFINITION
}
name={
isViewTableType
? t('label.view-definition')
: t('label.schema-definition')
}
/>
),
isHidden: isUndefined(tableDetails?.viewDefinition),
key: EntityTabs.VIEW_DEFINITION,
children: <QueryViewer sqlQuery={tableDetails?.viewDefinition ?? ''} />,
isHidden: isUndefined(tableDetails?.schemaDefinition),
key: isViewTableType
? EntityTabs.VIEW_DEFINITION
: EntityTabs.SCHEMA_DEFINITION,
children: (
<QueryViewer sqlQuery={tableDetails?.schemaDefinition ?? ''} />
),
},
{
label: (

2
openmetadata-ui/src/main/resources/ui/src/utils/DatasetDetailsUtils.ts
View File

@ -14,4 +14,4 @@
import { TabSpecificField } from '../enums/entity.enum';
// eslint-disable-next-line max-len
export const defaultFields = `${TabSpecificField.COLUMNS},${TabSpecificField.FOLLOWERS},${TabSpecificField.JOINS},${TabSpecificField.TAGS},${TabSpecificField.OWNER},${TabSpecificField.DATAMODEL},${TabSpecificField.TABLE_CONSTRAINTS},${TabSpecificField.VIEW_DEFINITION},${TabSpecificField.DOMAIN},${TabSpecificField.DATA_PRODUCTS},${TabSpecificField.VOTES},${TabSpecificField.EXTENSION}`;
export const defaultFields = `${TabSpecificField.COLUMNS},${TabSpecificField.FOLLOWERS},${TabSpecificField.JOINS},${TabSpecificField.TAGS},${TabSpecificField.OWNER},${TabSpecificField.DATAMODEL},${TabSpecificField.TABLE_CONSTRAINTS},${TabSpecificField.SCHEMA_DEFINITION},${TabSpecificField.DOMAIN},${TabSpecificField.DATA_PRODUCTS},${TabSpecificField.VOTES},${TabSpecificField.EXTENSION}`;