diff --git a/ingestion/src/metadata/ingestion/lineage/sql_lineage.py b/ingestion/src/metadata/ingestion/lineage/sql_lineage.py index 8f01d158aad..170364a9f3b 100644 --- a/ingestion/src/metadata/ingestion/lineage/sql_lineage.py +++ b/ingestion/src/metadata/ingestion/lineage/sql_lineage.py @@ -23,7 +23,7 @@ from metadata.generated.schema.type.entityLineage import ( ) from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.lineage.models import Dialect -from metadata.ingestion.lineage.parser import LineageParser +from metadata.ingestion.lineage.parser import LINEAGE_PARSING_TIMEOUT, LineageParser from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.utils import fqn from metadata.utils.fqn import build_es_fqn_search_string @@ -342,6 +342,7 @@ def get_lineage_by_query( schema_name: Optional[str], query: str, dialect: Dialect, + timeout_seconds: int = LINEAGE_PARSING_TIMEOUT, ) -> Optional[Iterator[AddLineageRequest]]: """ This method parses the query to get source, target and intermediate table names to create lineage, @@ -351,7 +352,7 @@ def get_lineage_by_query( try: logger.debug(f"Running lineage with query: {query}") - lineage_parser = LineageParser(query, dialect) + lineage_parser = LineageParser(query, dialect, timeout_seconds=timeout_seconds) raw_column_lineage = lineage_parser.column_lineage column_lineage.update(populate_column_lineage_map(raw_column_lineage)) @@ -405,6 +406,7 @@ def get_lineage_via_table_entity( service_name: str, query: str, dialect: Dialect, + timeout_seconds: int = LINEAGE_PARSING_TIMEOUT, ) -> Optional[Iterator[AddLineageRequest]]: """Get lineage from table entity @@ -427,7 +429,7 @@ def get_lineage_via_table_entity( try: logger.debug(f"Getting lineage via table entity using query: {query}") - lineage_parser = LineageParser(query, dialect) + lineage_parser = LineageParser(query, dialect, timeout_seconds=timeout_seconds) to_table_name = table_entity.name.__root__ for from_table_name in lineage_parser.source_tables: diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/query_parser.py b/ingestion/src/metadata/ingestion/source/database/bigquery/query_parser.py index f61a00336bb..88b5d197c89 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/query_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/query_parser.py @@ -57,7 +57,7 @@ class BigqueryQueryParserSource(QueryParserSource, ABC): start_time=start_time, end_time=end_time, region=self.service_connection.usageLocation, - filters=self.filters, + filters=self.get_filters(), result_limit=self.source_config.resultLimit, ) diff --git a/ingestion/src/metadata/ingestion/source/database/clickhouse/query_parser.py b/ingestion/src/metadata/ingestion/source/database/clickhouse/query_parser.py index 4c4833e4127..06f81e21ce8 100644 --- a/ingestion/src/metadata/ingestion/source/database/clickhouse/query_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/clickhouse/query_parser.py @@ -76,7 +76,7 @@ class ClickhouseQueryParserSource(QueryParserSource, ABC): return self.sql_stmt.format( start_time=start_time, end_time=end_time, - filters=self.filters, # pylint: disable=no-member + filters=self.get_filters(), result_limit=self.source_config.resultLimit, ) diff --git a/ingestion/src/metadata/ingestion/source/database/common_db_source.py b/ingestion/src/metadata/ingestion/source/database/common_db_source.py index c9cb4f18799..8c9304f07d5 100644 --- a/ingestion/src/metadata/ingestion/source/database/common_db_source.py +++ b/ingestion/src/metadata/ingestion/source/database/common_db_source.py @@ -436,6 +436,7 @@ class CommonDbSourceService( metadata=self.metadata, service_name=self.context.database_service.name.__root__, connection_type=self.service_connection.type.value, + timeout_seconds=self.source_config.viewParsingTimeoutLimit, ) def _get_foreign_constraints( diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py index 99293ec91b4..8bb59ba7781 100644 --- a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py @@ -640,6 +640,7 @@ class DbtSource(DbtServiceSource): database_name=source_elements[1], schema_name=source_elements[2], dialect=dialect, + timeout_seconds=self.source_config.parsingTimeoutLimit, ) for lineage_request in lineages or []: yield lineage_request diff --git a/ingestion/src/metadata/ingestion/source/database/lineage_source.py b/ingestion/src/metadata/ingestion/source/database/lineage_source.py index 2528c9b348e..44ec707d33c 100644 --- a/ingestion/src/metadata/ingestion/source/database/lineage_source.py +++ b/ingestion/src/metadata/ingestion/source/database/lineage_source.py @@ -113,6 +113,7 @@ class LineageSource(QueryParserSource, ABC): database_name=table_query.databaseName, schema_name=table_query.databaseSchema, dialect=dialect, + timeout_seconds=self.source_config.parsingTimeoutLimit, ) for lineage_request in lineages or []: diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/query_parser.py b/ingestion/src/metadata/ingestion/source/database/postgres/query_parser.py index 5759c943fe8..38aae409bbb 100644 --- a/ingestion/src/metadata/ingestion/source/database/postgres/query_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/postgres/query_parser.py @@ -76,7 +76,7 @@ class PostgresQueryParserSource(QueryParserSource, ABC): """ return self.sql_stmt.format( result_limit=self.config.sourceConfig.config.resultLimit, - filters=self.filters, + filters=self.get_filters(), time_column_name=self.get_postgres_time_column_name(), ) diff --git a/ingestion/src/metadata/ingestion/source/database/query_parser_source.py b/ingestion/src/metadata/ingestion/source/database/query_parser_source.py index 536aa088e30..22c4710a97e 100644 --- a/ingestion/src/metadata/ingestion/source/database/query_parser_source.py +++ b/ingestion/src/metadata/ingestion/source/database/query_parser_source.py @@ -103,10 +103,15 @@ class QueryParserSource(Source[Union[TableQuery, AddLineageRequest]], ABC): return self.sql_stmt.format( start_time=start_time, end_time=end_time, - filters=self.filters, + filters=self.get_filters(), result_limit=self.source_config.resultLimit, ) + def get_filters(self) -> str: + if self.source_config.filterCondition: + return f"{self.filters} AND {self.source_config.filterCondition}" + return self.filters + def close(self): """ By default, there is nothing to close diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/query_parser.py b/ingestion/src/metadata/ingestion/source/database/redshift/query_parser.py index 8a70265454c..d57e5b16dd0 100644 --- a/ingestion/src/metadata/ingestion/source/database/redshift/query_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/redshift/query_parser.py @@ -54,6 +54,6 @@ class RedshiftQueryParserSource(QueryParserSource, ABC): return self.sql_stmt.format( start_time=start_time, end_time=end_time, - filters=self.filters, + filters=self.get_filters(), result_limit=self.source_config.resultLimit, ) diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/query_parser.py b/ingestion/src/metadata/ingestion/source/database/snowflake/query_parser.py index 2778fa926ea..f50684d4582 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/query_parser.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/query_parser.py @@ -60,7 +60,7 @@ class SnowflakeQueryParserSource(QueryParserSource, ABC): start_time=start_time, end_time=end_time, result_limit=self.config.sourceConfig.config.resultLimit, - filters=self.filters, + filters=self.get_filters(), ) def set_session_query_tag(self) -> None: diff --git a/ingestion/src/metadata/utils/db_utils.py b/ingestion/src/metadata/utils/db_utils.py index 0896ecf593c..10258243c81 100644 --- a/ingestion/src/metadata/utils/db_utils.py +++ b/ingestion/src/metadata/utils/db_utils.py @@ -17,7 +17,7 @@ import traceback from metadata.generated.schema.entity.data.table import Table from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper -from metadata.ingestion.lineage.parser import LineageParser +from metadata.ingestion.lineage.parser import LINEAGE_PARSING_TIMEOUT, LineageParser from metadata.ingestion.lineage.sql_lineage import ( get_lineage_by_query, get_lineage_via_table_entity, @@ -39,7 +39,11 @@ def get_host_from_host_port(uri: str) -> str: def get_view_lineage( - view: TableView, metadata: OpenMetadata, service_name: str, connection_type: str + view: TableView, + metadata: OpenMetadata, + service_name: str, + connection_type: str, + timeout_seconds: int = LINEAGE_PARSING_TIMEOUT, ): """ Method to generate view lineage @@ -64,7 +68,9 @@ def get_view_lineage( try: connection_type = str(connection_type) dialect = ConnectionTypeDialectMapper.dialect_of(connection_type) - lineage_parser = LineageParser(view_definition, dialect) + lineage_parser = LineageParser( + view_definition, dialect, timeout_seconds=timeout_seconds + ) if lineage_parser.source_tables and lineage_parser.target_tables: yield from get_lineage_by_query( metadata, @@ -73,6 +79,7 @@ def get_view_lineage( database_name=db_name, schema_name=schema_name, dialect=dialect, + timeout_seconds=timeout_seconds, ) or [] else: @@ -84,6 +91,7 @@ def get_view_lineage( schema_name=schema_name, query=view_definition, dialect=dialect, + timeout_seconds=timeout_seconds, ) or [] except Exception as exc: logger.debug(traceback.format_exc()) diff --git a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/ingestion/workflows/lineage/filter-query-set.md b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/ingestion/workflows/lineage/filter-query-set.md new file mode 100644 index 00000000000..bdb413f17e0 --- /dev/null +++ b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/ingestion/workflows/lineage/filter-query-set.md @@ -0,0 +1,213 @@ +--- +title: Lineage Query Filtering +slug: /connectors/ingestion/workflows/lineage/filter-query-set +--- + +# Lineage Query Filtering + +In order to extract usage information, OpenMetadata parses the queries that have run against the database. We fetch these queries from the query history table of the respective data source. This query fetches all the queries executed within configured no of days. In this guide we we understand how we can filter out the query history result set, this can help to exclude queries with specific pattern or queries executed on a specific schema or database, depending on the data source. + +Query filtering is supported for both Usage & Lineage workflows. While configuring either usage or lineage workflow you will find a `Filtering Condition` text field where you can provide the sql condition which will be added to already existing conditions with an `AND` operation. In later part of this document you will find how to write this condition for supported data source. + +{% image + src="/images/v1.1.1/features/ingestion/workflows/lineage/filter-condition-field.png" + alt="filter-condition-field" + caption="Filter Condition Field" + /%} + + +## Snowflake Filter Condition + +To fetch the query history log from snowflake we execute the following query + +``` +SELECT + query_type, + query_text, + user_name, + database_name, + schema_name, + start_time, + end_time, + total_elapsed_time/1000 duration + from snowflake.account_usage.query_history + WHERE query_text NOT LIKE '/* {app": "OpenMetadata", %} */%' + AND query_text NOT LIKE '/* {"app": "dbt", %} */%' + AND start_time between to_timestamp_ltz('{start_time}') and to_timestamp_ltz('{end_time}') + AND QUERY_TYPE NOT IN ('ROLLBACK','CREATE_USER',....) + + AND {**your filter condition**} + + LIMIT {result_limit} +``` + +You can refer to [this snowflake documentation](https://docs.snowflake.com/en/sql-reference/functions/query_history) to find out more about the query history table. + +For example if you want to add a condition to filter out queries executed by metabase client, i.e. the queries staring with `-- metabase %` then you can put the condition as `query_text NOT LIKE '--metabase %'` + +you also need to further restrict the query log and need only queries which have been executed on `SALES` database then you can put the filter condition as `query_text NOT LIKE '--metabase %' AND database_name='SALES'`. + + +## Bigquery Filter Condition + +To fetch the query history log from bigquery we execute the following query + +``` +SELECT + project_id as database_name, + user_email as user_name, + statement_type as query_type, + start_time, + end_time, + query as query_text, + null as schema_name, + total_slot_ms/1000 as duration +FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT +WHERE creation_time BETWEEN "{start_time}" AND "{end_time}" + AND statement_type IN ("SELECT",....) + + AND {**your filter condition**} + + AND job_type = "QUERY" + AND state = "DONE" + AND IFNULL(statement_type, "NO") not in ("NO", "DROP_TABLE", "CREATE_TABLE") + AND query NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' + AND query NOT LIKE '/* {{"app": "dbt", %%}} */%%' + LIMIT {result_limit} +``` + +You can refer to [this bigquery documentation](https://cloud.google.com/bigquery/docs/information-schema-jobs) to find out more about the JOBS_BY_PROJECT table. + +For example if you want to add a condition to filter out queries executed by metabase client, i.e. the queries staring with `-- metabase %` then you can put the condition as `query NOT LIKE '--metabase %'`. + + +## MSSQL Filter Condition + +To fetch the query history log from MSSQL we execute the following query + +``` +SELECT TOP {result_limit} + db.NAME database_name, + t.text query_text, + s.last_execution_time start_time, + DATEADD(s, s.total_elapsed_time/1000, s.last_execution_time) end_time, + s.total_elapsed_time/1000 duration, + NULL schema_name, + NULL query_type, + NULL user_name, + NULL aborted +FROM sys.dm_exec_cached_plans AS p +INNER JOIN sys.dm_exec_query_stats AS s +ON p.plan_handle = s.plan_handle +CROSS APPLY sys.Dm_exec_sql_text(p.plan_handle) AS t +INNER JOIN sys.databases db +ON db.database_id = t.dbid +WHERE s.last_execution_time between '{start_time}' and '{end_time}' + AND t.text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' + AND t.text NOT LIKE '/* {{"app": "dbt", %%}} */%%' + AND p.objtype != 'Prepared' + + AND {**your filter condition**} + +ORDER BY s.last_execution_time DESC +``` + +You can refer to [this mssql documentation](https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-exec-cached-plans-transact-sql?view=sql-server-ver16) to find out more about the dm_exec_cached_plans table. + +For example if you want to add a condition to filter out queries executed by metabase client, i.e. the queries staring with `-- metabase %` then you can put the condition as `t.text NOT LIKE '--metabase %'`. + +you also need to further restrict the query log and need only queries which have been executed on `SALES` database then you can put the filter condition as `t.text NOT LIKE '--metabase %' AND db.NAME='SALES'`. + + +## Clickhouse Filter Condition + +To fetch the query history log from clickhouse we execute the following query + +``` +Select + query_start_time start_time, + DATEADD(query_duration_ms, query_start_time) end_time, + query_duration_ms/1000 duration, + 'default' database_name, + user user_name, + FALSE aborted, + query_id query_id, + query query_text, + databases schema_name, + tables tables +From system.query_log +Where start_time between '{start_time}' and '{end_time}' +and CAST(type,'Int8') <> 3 +and CAST(type,'Int8') <> 4 +and query NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' +and query NOT LIKE '/* {{"app": "dbt", %%}} */%%' + +AND {**your filter condition**} + +and (`type`='QueryFinish' or `type`='QueryStart') +LIMIT {result_limit} +``` + +You can refer to [this clickhouse documentation](https://clickhouse.com/docs/en/operations/system-tables/query_log) to find out more about the query_log table. + +For example if you want to add a condition to filter out queries executed by metabase client, i.e. the queries staring with `-- metabase %` then you can put the condition as `query NOT LIKE '--metabase %'`. + + +## Vertica Filter Condition + +To fetch the query history log from vertica we execute the following query + +``` +SELECT + DBNAME() AS database_name, + p.query AS query_text, + r.start_timestamp AS start_time, + r.end_timestamp AS end_time, + p.schema_name, + p.query_duration_us/1000 AS duration, + p.query_type, + p.user_name, + NULL aborted + FROM query_profiles p + LEFT JOIN query_requests r + ON p.TRANSACTION_ID = r.TRANSACTION_ID + AND p.STATEMENT_ID = r.STATEMENT_ID + WHERE query_start between '{start_time}' and '{end_time}' + AND query NOT LIKE '%%/* {{"app": "OpenMetadata", %%}} */%%' + AND query NOT LIKE '/* {{"app": "dbt", %%}} */%%' + AND success = 1 + + AND {**your filter condition**} + + ORDER BY query_start DESC + LIMIT {result_limit} +``` + +You can refer to [this vertica documentation](https://www.vertica.com/docs/10.0.x/HTML/Content/Authoring/SQLReferenceManual/SystemTables/MONITOR/QUERY_PROFILES.htm) to find out more about the query_profiles table. + +For example if you want to add a condition to filter out queries executed by metabase client, i.e. the queries staring with `-- metabase %` then you can put the condition as `query NOT LIKE '--metabase %'`. + + +## Redshift Filter Condition + +To fetch the query history log from redshift we execute the following query + +``` +SELECT * + FROM pg_catalog.stl_query +WHERE userid > 1 + + AND {**your filter condition**} + + -- Filter out all automated & cursor queries + AND querytxt NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' + AND querytxt NOT LIKE '/* {{"app": "dbt", %%}} */%%' + AND aborted = 0 + AND starttime >= '{start_time}' + AND starttime < '{end_time}' + LIMIT {result_limit} +``` + +You can refer to [this redshift documentation](https://docs.aws.amazon.com/redshift/latest/dg/r_STL_QUERY.html) to find out more about the stl_query table. + +For example if you want to add a condition to filter out queries executed by metabase client, i.e. the queries staring with `-- metabase %` then you can put the condition as `query NOT LIKE '--metabase %'`. diff --git a/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/ingestion/workflows/usage/filter-query-set.md b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/ingestion/workflows/usage/filter-query-set.md new file mode 100644 index 00000000000..51906f377e9 --- /dev/null +++ b/openmetadata-docs/content/v1.1.1-SNAPSHOT/connectors/ingestion/workflows/usage/filter-query-set.md @@ -0,0 +1,213 @@ +--- +title: Usage Query Filtering +slug: /connectors/ingestion/workflows/usage/filter-query-set +--- + +# Usage Query Filtering + +In order to extract usage information, OpenMetadata parses the queries that have run against the database. We fetch these queries from the query history table of the respective data source. This query fetches all the queries executed within configured no of days. In this guide we we understand how we can filter out the query history result set, this can help to exclude queries with specific pattern or queries executed on a specific schema or database, depending on the data source. + +Query filtering is supported for both Usage & Lineage workflows. While configuring either usage or lineage workflow you will find a `Filtering Condition` text field where you can provide the sql condition which will be added to already existing conditions with an `AND` operation. In later part of this document you will find how to write this condition for supported data source. + +{% image + src="/images/v1.1.1/features/ingestion/workflows/usage/filter-condition-field.png" + alt="filter-condition-field" + caption="Filter Condition Field" + /%} + + +## Snowflake Filter Condition + +To fetch the query history log from snowflake we execute the following query + +``` +SELECT + query_type, + query_text, + user_name, + database_name, + schema_name, + start_time, + end_time, + total_elapsed_time/1000 duration + from snowflake.account_usage.query_history + WHERE query_text NOT LIKE '/* {app": "OpenMetadata", %} */%' + AND query_text NOT LIKE '/* {"app": "dbt", %} */%' + AND start_time between to_timestamp_ltz('{start_time}') and to_timestamp_ltz('{end_time}') + AND QUERY_TYPE NOT IN ('ROLLBACK','CREATE_USER',....) + + AND {**your filter condition**} + + LIMIT {result_limit} +``` + +You can refer to [this snowflake documentation](https://docs.snowflake.com/en/sql-reference/functions/query_history) to find out more about the query history table. + +For example if you want to add a condition to filter out queries executed by metabase client, i.e. the queries staring with `-- metabase %` then you can put the condition as `query_text NOT LIKE '--metabase %'` + +you also need to further restrict the query log and need only queries which have been executed on `SALES` database then you can put the filter condition as `query_text NOT LIKE '--metabase %' AND database_name='SALES'`. + + +## Bigquery Filter Condition + +To fetch the query history log from bigquery we execute the following query + +``` +SELECT + project_id as database_name, + user_email as user_name, + statement_type as query_type, + start_time, + end_time, + query as query_text, + null as schema_name, + total_slot_ms/1000 as duration +FROM `region-{region}`.INFORMATION_SCHEMA.JOBS_BY_PROJECT +WHERE creation_time BETWEEN "{start_time}" AND "{end_time}" + AND statement_type IN ("SELECT",....) + + AND {**your filter condition**} + + AND job_type = "QUERY" + AND state = "DONE" + AND IFNULL(statement_type, "NO") not in ("NO", "DROP_TABLE", "CREATE_TABLE") + AND query NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' + AND query NOT LIKE '/* {{"app": "dbt", %%}} */%%' + LIMIT {result_limit} +``` + +You can refer to [this bigquery documentation](https://cloud.google.com/bigquery/docs/information-schema-jobs) to find out more about the JOBS_BY_PROJECT table. + +For example if you want to add a condition to filter out queries executed by metabase client, i.e. the queries staring with `-- metabase %` then you can put the condition as `query NOT LIKE '--metabase %'`. + + +## MSSQL Filter Condition + +To fetch the query history log from MSSQL we execute the following query + +``` +SELECT TOP {result_limit} + db.NAME database_name, + t.text query_text, + s.last_execution_time start_time, + DATEADD(s, s.total_elapsed_time/1000, s.last_execution_time) end_time, + s.total_elapsed_time/1000 duration, + NULL schema_name, + NULL query_type, + NULL user_name, + NULL aborted +FROM sys.dm_exec_cached_plans AS p +INNER JOIN sys.dm_exec_query_stats AS s +ON p.plan_handle = s.plan_handle +CROSS APPLY sys.Dm_exec_sql_text(p.plan_handle) AS t +INNER JOIN sys.databases db +ON db.database_id = t.dbid +WHERE s.last_execution_time between '{start_time}' and '{end_time}' + AND t.text NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' + AND t.text NOT LIKE '/* {{"app": "dbt", %%}} */%%' + AND p.objtype != 'Prepared' + + AND {**your filter condition**} + +ORDER BY s.last_execution_time DESC +``` + +You can refer to [this mssql documentation](https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-exec-cached-plans-transact-sql?view=sql-server-ver16) to find out more about the dm_exec_cached_plans table. + +For example if you want to add a condition to filter out queries executed by metabase client, i.e. the queries staring with `-- metabase %` then you can put the condition as `t.text NOT LIKE '--metabase %'`. + +you also need to further restrict the query log and need only queries which have been executed on `SALES` database then you can put the filter condition as `t.text NOT LIKE '--metabase %' AND db.NAME='SALES'`. + + +## Clickhouse Filter Condition + +To fetch the query history log from clickhouse we execute the following query + +``` +Select + query_start_time start_time, + DATEADD(query_duration_ms, query_start_time) end_time, + query_duration_ms/1000 duration, + 'default' database_name, + user user_name, + FALSE aborted, + query_id query_id, + query query_text, + databases schema_name, + tables tables +From system.query_log +Where start_time between '{start_time}' and '{end_time}' +and CAST(type,'Int8') <> 3 +and CAST(type,'Int8') <> 4 +and query NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' +and query NOT LIKE '/* {{"app": "dbt", %%}} */%%' + +AND {**your filter condition**} + +and (`type`='QueryFinish' or `type`='QueryStart') +LIMIT {result_limit} +``` + +You can refer to [this clickhouse documentation](https://clickhouse.com/docs/en/operations/system-tables/query_log) to find out more about the query_log table. + +For example if you want to add a condition to filter out queries executed by metabase client, i.e. the queries staring with `-- metabase %` then you can put the condition as `query NOT LIKE '--metabase %'`. + + +## Vertica Filter Condition + +To fetch the query history log from vertica we execute the following query + +``` +SELECT + DBNAME() AS database_name, + p.query AS query_text, + r.start_timestamp AS start_time, + r.end_timestamp AS end_time, + p.schema_name, + p.query_duration_us/1000 AS duration, + p.query_type, + p.user_name, + NULL aborted + FROM query_profiles p + LEFT JOIN query_requests r + ON p.TRANSACTION_ID = r.TRANSACTION_ID + AND p.STATEMENT_ID = r.STATEMENT_ID + WHERE query_start between '{start_time}' and '{end_time}' + AND query NOT LIKE '%%/* {{"app": "OpenMetadata", %%}} */%%' + AND query NOT LIKE '/* {{"app": "dbt", %%}} */%%' + AND success = 1 + + AND {**your filter condition**} + + ORDER BY query_start DESC + LIMIT {result_limit} +``` + +You can refer to [this vertica documentation](https://www.vertica.com/docs/10.0.x/HTML/Content/Authoring/SQLReferenceManual/SystemTables/MONITOR/QUERY_PROFILES.htm) to find out more about the query_profiles table. + +For example if you want to add a condition to filter out queries executed by metabase client, i.e. the queries staring with `-- metabase %` then you can put the condition as `query NOT LIKE '--metabase %'`. + + +## Redshift Filter Condition + +To fetch the query history log from redshift we execute the following query + +``` +SELECT * + FROM pg_catalog.stl_query +WHERE userid > 1 + + AND {**your filter condition**} + + -- Filter out all automated & cursor queries + AND querytxt NOT LIKE '/* {{"app": "OpenMetadata", %%}} */%%' + AND querytxt NOT LIKE '/* {{"app": "dbt", %%}} */%%' + AND aborted = 0 + AND starttime >= '{start_time}' + AND starttime < '{end_time}' + LIMIT {result_limit} +``` + +You can refer to [this redshift documentation](https://docs.aws.amazon.com/redshift/latest/dg/r_STL_QUERY.html) to find out more about the stl_query table. + +For example if you want to add a condition to filter out queries executed by metabase client, i.e. the queries staring with `-- metabase %` then you can put the condition as `query NOT LIKE '--metabase %'`. diff --git a/openmetadata-docs/content/v1.1.1-SNAPSHOT/menu.md b/openmetadata-docs/content/v1.1.1-SNAPSHOT/menu.md index a37d792e7a1..36461582de3 100644 --- a/openmetadata-docs/content/v1.1.1-SNAPSHOT/menu.md +++ b/openmetadata-docs/content/v1.1.1-SNAPSHOT/menu.md @@ -513,10 +513,14 @@ site_menu: url: /connectors/ingestion/workflows/usage - category: Connectors / Ingestion / Workflows / Usage / Usage Workflow Through Query Logs url: /connectors/ingestion/workflows/usage/usage-workflow-query-logs + - category: Connectors / Ingestion / Workflows / Usage / Usage Query Filtering + url: /connectors/ingestion/workflows/usage/filter-query-set - category: Connectors / Ingestion / Workflows / Lineage url: /connectors/ingestion/workflows/lineage - category: Connectors / Ingestion / Workflows / Lineage / Lineage Workflow Through Query Logs url: /connectors/ingestion/workflows/lineage/lineage-workflow-query-logs + - category: Connectors / Ingestion / Workflows / Lineage / Lineage Query Filtering + url: /connectors/ingestion/workflows/lineage/filter-query-set - category: Connectors / Ingestion / Workflows / dbt url: /connectors/ingestion/workflows/dbt - category: Connectors / Ingestion / Workflows / dbt / Ingest dbt UI diff --git a/openmetadata-docs/images/v1.1.1/features/ingestion/workflows/lineage/filter-condition-field.png b/openmetadata-docs/images/v1.1.1/features/ingestion/workflows/lineage/filter-condition-field.png new file mode 100644 index 00000000000..9d5b5cd5780 Binary files /dev/null and b/openmetadata-docs/images/v1.1.1/features/ingestion/workflows/lineage/filter-condition-field.png differ diff --git a/openmetadata-docs/images/v1.1.1/features/ingestion/workflows/usage/filter-condition-field.png b/openmetadata-docs/images/v1.1.1/features/ingestion/workflows/usage/filter-condition-field.png new file mode 100644 index 00000000000..ad56e310670 Binary files /dev/null and b/openmetadata-docs/images/v1.1.1/features/ingestion/workflows/usage/filter-condition-field.png differ diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json index 46d4475bf18..78df38ebb5f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceMetadataPipeline.json @@ -48,6 +48,11 @@ "type": "boolean", "default": false }, + "viewParsingTimeoutLimit": { + "description": "Configuration to set the timeout for parsing view query in seconds.", + "type": "integer", + "default": "300" + }, "schemaFilterPattern": { "description": "Regex to only fetch tables or databases that matches the pattern.", "$ref": "../type/filterPattern.json#/definitions/filterPattern" diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceQueryLineagePipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceQueryLineagePipeline.json index 40ddd6e50bd..9b5b423079c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceQueryLineagePipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceQueryLineagePipeline.json @@ -31,6 +31,15 @@ "type": "integer", "default": "1000" }, + "parsingTimeoutLimit": { + "description": "Configuration to set the timeout for parsing the query in seconds.", + "type": "integer", + "default": "300" + }, + "filterCondition": { + "description": "Configuration the condition to filter the query history.", + "type": "string" + }, "schemaFilterPattern": { "description": "Regex to only fetch tables or databases that matches the pattern.", "$ref": "../type/filterPattern.json#/definitions/filterPattern" diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceQueryUsagePipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceQueryUsagePipeline.json index 458d59b2849..226e642fa97 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceQueryUsagePipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceQueryUsagePipeline.json @@ -27,6 +27,10 @@ "type": "string", "default": "/tmp/query_log" }, + "filterCondition": { + "description": "Configuration the condition to filter the query history.", + "type": "string" + }, "resultLimit": { "description": "Configuration to set the limit for query logs", "type": "integer", diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtPipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtPipeline.json index 28b19235701..207a735a37c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtPipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/dbtPipeline.json @@ -66,6 +66,11 @@ "description": "Regex exclude tables or databases that matches the pattern.", "$ref": "../type/filterPattern.json#/definitions/filterPattern" }, + "parsingTimeoutLimit": { + "description": "Configuration to set the timeout for parsing the query in seconds.", + "type": "integer", + "default": "300" + }, "databaseFilterPattern": { "description": "Regex to only fetch databases that matches the pattern.", "$ref": "../type/filterPattern.json#/definitions/filterPattern" diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/dbt.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/dbt.md index 8db01db501a..35b34b0558a 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/dbt.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/dbt.md @@ -343,3 +343,10 @@ Option to include fetching the tags metadata from dbt. When enabled, OpenMetadata will fetch tags associated with tables and columns from dbt `manifest.json` and attach them to the corresponding tables in OpenMetadata. $$ + + +$$section +### Query Parsing Timeout Limit $(id="parsingTimeoutLimit") + +Specify the timeout limit for parsing the sql queries to perform the lineage analysis. +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/lineage.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/lineage.md index 36d36d7f39a..bd6e0a98d25 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/lineage.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/lineage.md @@ -31,3 +31,20 @@ SELECT xyz FROM query_history limit This value will take precedence over the `Query Log Duration`. $$ + + +$$section +### Filtering Condition $(id="filterCondition") + +We execute a query on query history table of the respective data source to perform the query analysis and extract the lineage and usage information. This field will be useful when you want to restrict some queries from being part of this analysis. In this field you can specify a sql condition that will be applied on the query history result set. + +For example: `query_text not ilike '--- metabase query %'` + +Checkout [this](https://docs.open-metadata.org/connectors/ingestion/workflows/usage/filter-query-set) document for further examples on filter conditions. +$$ + +$$section +### Query Parsing Timeout Limit $(id="parsingTimeoutLimit") + +Specify the timeout limit for parsing the sql queries to perform the lineage analysis. +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md index a880208a3a5..6b39e9650c1 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/metadata.md @@ -120,3 +120,10 @@ Here are some examples of scenarios where tables will get soft deleted if this f - If you already have `SchemaA` & `SchemaB` ingested in OpenMetadata ,then later you apply a `Schema Filter Pattern` to exclude `SchemaB`, ALL tables from `SchemaB` will be deleted due to this ingestion pipeline. This might be useful if you want to remove a full schema from OpenMetadata that you missed to filter out the first time. $$ + + +$$section +### View Definition Parsing Timeout Limit $(id="viewParsingTimeoutLimit") + +Specify the timeout limit for parsing the view definition sql queries to perform the lineage analysis. +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/usage.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/usage.md index 3a9c9a136a8..bc6e3225af5 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/usage.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/workflows/usage.md @@ -33,3 +33,15 @@ SELECT xyz FROM query_history limit This value will take precedence over the `Query Log Duration`. $$ + + +$$section +### Filtering Condition $(id="filterCondition") + +We execute a query on query history table of the respective data source to perform the query analysis and extract the lineage and usage information. This field will be useful when you want to restrict some queries from being part of this analysis. In this field you can specify a sql condition that will be applied on the query history result set. + +For example: `query_text not ilike '--- metabase query %'` + + +Checkout [this](https://docs.open-metadata.org/connectors/ingestion/workflows/usage/filter-query-set) document for further examples on filter conditions. +$$ \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 7df8fa101f2..f5b010fce08 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -24,6 +24,7 @@ import React, { import { useTranslation } from 'react-i18next'; import { DBT_CLASSIFICATION_DEFAULT_VALUE, + DEFAULT_PARSING_TIMEOUT_LIMIT, INITIAL_FILTER_PATTERN, STEPS_FOR_ADD_INGESTION, } from '../../constants/Ingestions.constant'; @@ -251,6 +252,11 @@ const AddIngestion = ({ confidence: sourceConfig?.confidence, dbtClassificationName: sourceConfig?.dbtClassificationName ?? DBT_CLASSIFICATION_DEFAULT_VALUE, // default value from Json Schema + parsingTimeoutLimit: + sourceConfig?.parsingTimeoutLimit ?? DEFAULT_PARSING_TIMEOUT_LIMIT, + viewParsingTimeoutLimit: + sourceConfig?.viewParsingTimeoutLimit ?? DEFAULT_PARSING_TIMEOUT_LIMIT, + filterCondition: sourceConfig?.filterCondition ?? '', }), [] ); @@ -395,6 +401,7 @@ const AddIngestion = ({ topicFilterPattern, useFqnFilter, includeOwners, + viewParsingTimeoutLimit, } = state; switch (serviceCategory) { @@ -418,6 +425,7 @@ const AddIngestion = ({ markDeletedTables: markDeletedTables, markAllDeletedTables: markAllDeletedTables, type: ConfigType.DatabaseMetadata, + viewParsingTimeoutLimit: viewParsingTimeoutLimit, }; } case ServiceCategory.MESSAGING_SERVICES: { @@ -513,6 +521,8 @@ const AddIngestion = ({ timeoutSeconds, processPii, confidence, + filterCondition, + parsingTimeoutLimit, } = state; switch (type) { case PipelineType.Usage: { @@ -521,6 +531,7 @@ const AddIngestion = ({ resultLimit: resultLimit, stageFileLocation: stageFileLocation, type: usageIngestionType, + filterCondition: filterCondition, }; } case PipelineType.Lineage: { @@ -528,6 +539,8 @@ const AddIngestion = ({ queryLogDuration: queryLogDuration, resultLimit: resultLimit, type: lineageIngestionType, + filterCondition: filterCondition, + parsingTimeoutLimit: parsingTimeoutLimit, }; } case PipelineType.Profiler: { @@ -572,6 +585,7 @@ const AddIngestion = ({ databaseFilterPattern: databaseFilterPattern, schemaFilterPattern: schemaFilterPattern, tableFilterPattern: tableFilterPattern, + parsingTimeoutLimit: parsingTimeoutLimit, }; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx index ec14e897c8f..9f9390de8e2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx @@ -95,6 +95,9 @@ const ConfigureIngestion = ({ processPii, confidence, includeOwners, + parsingTimeoutLimit, + filterCondition, + viewParsingTimeoutLimit, } = useMemo( () => ({ dataModelFilterPattern: data.dataModelFilterPattern, @@ -143,6 +146,9 @@ const ConfigureIngestion = ({ markDeletedMlModels: data.markDeletedMlModels, markDeletedPipelines: data.markDeletedPipelines, confidence: data.confidence, + parsingTimeoutLimit: data.parsingTimeoutLimit, + filterCondition: data.filterCondition, + viewParsingTimeoutLimit: data.viewParsingTimeoutLimit, }), [data] ); @@ -218,6 +224,14 @@ const ConfigureIngestion = ({ const handleConfidenceScore = handleIntValue('confidence'); + const handleParsingTimeoutLimit = handleIntValue('parsingTimeoutLimit'); + + const handleViewParsingTimeoutLimit = handleIntValue( + 'viewParsingTimeoutLimit' + ); + + const handleFilterCondition = handleValueChange('filterCondition'); + const handleProfileSampleTypeChange = (value: ProfileSampleType) => { onChange({ profileSampleType: value, @@ -252,6 +266,24 @@ const ConfigureIngestion = ({ ], }, ]; + + const filterConditionField: FieldProp = { + name: 'filterCondition', + label: t('label.filtering-condition'), + type: FieldTypes.TEXT, + required: false, + id: 'root/filterCondition', + hasSeparator: true, + formItemProps: { + initialValue: filterCondition, + }, + props: { + 'data-testid': 'filtering-condition', + value: filterCondition, + onChange: handleFilterCondition, + }, + }; + const includeOwnersField: FieldProp = { name: 'includeOwners', label: t('label.include-owner'), @@ -563,6 +595,22 @@ const ConfigureIngestion = ({ }, ] as FieldProp[]) : []), + { + name: 'viewParsingTimeoutLimit', + label: t('label.view-parsing-timeout-limit'), + type: FieldTypes.NUMBER, + required: false, + id: 'root/viewParsingTimeoutLimit', + hasSeparator: true, + formItemProps: { + initialValue: viewParsingTimeoutLimit, + }, + props: { + 'data-testid': 'dbt-view-parsing-timeout-limit', + value: viewParsingTimeoutLimit, + onChange: handleViewParsingTimeoutLimit, + }, + }, ]; const dashboardMetadataFields: FieldProp[] = [ @@ -899,6 +947,7 @@ const ConfigureIngestion = ({ }, }, rateLimitField, + filterConditionField, loggerLevelField, ]; @@ -910,6 +959,23 @@ const ConfigureIngestion = ({ queryLogDurationField, rateLimitField, loggerLevelField, + filterConditionField, + { + name: 'parsingTimeoutLimit', + label: t('label.parsing-timeout-limit'), + type: FieldTypes.NUMBER, + required: false, + id: 'root/parsingTimeoutLimit', + hasSeparator: true, + formItemProps: { + initialValue: parsingTimeoutLimit, + }, + props: { + 'data-testid': 'dbt-parsing-timeout-limit', + value: parsingTimeoutLimit, + onChange: handleParsingTimeoutLimit, + }, + }, ]; return generateFormFields(fields); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts index 6b8f4810e88..0118b1849a2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts @@ -114,6 +114,7 @@ export interface AddIngestionState { dbtConfigSourceType: DBT_SOURCES; description: string; enableDebugLog: boolean; + filterCondition: string; gcsConfigType: GCS_CONFIG | undefined; includeLineage: boolean; includeTags: boolean; @@ -153,6 +154,8 @@ export interface AddIngestionState { timeoutSeconds: number; topicFilterPattern: FilterPattern; useFqnFilter: boolean; + viewParsingTimeoutLimit: number; + parsingTimeoutLimit: number; processPii: boolean; includeOwners: boolean; confidence?: number; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCloudConfig.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCloudConfig.test.tsx index 80648add146..c281a7fd8ab 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCloudConfig.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCloudConfig.test.tsx @@ -14,6 +14,7 @@ import { render, screen } from '@testing-library/react'; import React from 'react'; import { DBTCloudConfig } from './DBTCloudConfig'; +import { dbtParsingTimeoutLimit } from './DBTFormConstants'; const mockProps = { dbtCloudAccountId: '', @@ -21,6 +22,7 @@ const mockProps = { dbtUpdateDescriptions: false, dbtCloudUrl: 'https://cloud.getdbt.com/', enableDebugLog: false, + parsingTimeoutLimit: dbtParsingTimeoutLimit, }; jest.mock('./DBTCommonFields.component', () => diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCloudConfig.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCloudConfig.tsx index 87542176aca..23ba0c9f6d3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCloudConfig.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCloudConfig.tsx @@ -20,6 +20,7 @@ import { DbtConfigCloud } from './DBTConfigForm.interface'; interface Props extends DbtConfigCloud { enableDebugLog: boolean; + parsingTimeoutLimit: number; } export const DBTCloudConfig: FunctionComponent = ({ @@ -32,6 +33,7 @@ export const DBTCloudConfig: FunctionComponent = ({ dbtCloudUrl = 'https://cloud.getdbt.com/', dbtClassificationName, enableDebugLog, + parsingTimeoutLimit, }: Props) => { const cloudConfigFields: FieldProp[] = [ { @@ -110,6 +112,7 @@ export const DBTCloudConfig: FunctionComponent = ({ descriptionId="cloud-update-description" enableDebugLog={enableDebugLog} includeTags={includeTags} + parsingTimeoutLimit={parsingTimeoutLimit} /> ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCommonFields.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCommonFields.component.tsx index 50cbd10db09..622a63c150a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCommonFields.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCommonFields.component.tsx @@ -22,6 +22,7 @@ interface Props { dbtUpdateDescriptions: boolean; enableDebugLog: boolean; includeTags: boolean; + parsingTimeoutLimit: number; } function DBTCommonFields({ @@ -30,6 +31,7 @@ function DBTCommonFields({ dbtClassificationName, enableDebugLog, includeTags, + parsingTimeoutLimit, }: Props) { const { t } = useTranslation(); @@ -100,6 +102,20 @@ function DBTCommonFields({ valuePropName: 'checked', }, }, + { + name: 'parsingTimeoutLimit', + label: t('label.parsing-timeout-limit'), + type: FieldTypes.NUMBER, + required: false, + props: { + 'data-testid': 'dbt-parsing-timeout-limit', + }, + id: 'root/parsingTimeoutLimit', + hasSeparator: true, + formItemProps: { + initialValue: parsingTimeoutLimit, + }, + }, ]; return {generateFormFields(commonFields)}; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCommonFields.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCommonFields.test.tsx index e3f43853be2..249689f39e5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCommonFields.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTCommonFields.test.tsx @@ -14,6 +14,7 @@ import { render, screen } from '@testing-library/react'; import React from 'react'; import DBTCommonFields from './DBTCommonFields.component'; +import { dbtParsingTimeoutLimit } from './DBTFormConstants'; const mockProps = { dbtUpdateDescriptions: false, @@ -21,6 +22,7 @@ const mockProps = { descriptionId: 'test-id', dbtClassificationName: 'DBT', enableDebugLog: false, + parsingTimeoutLimit: dbtParsingTimeoutLimit, }; describe('DBTCommonFields', () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.tsx index 3539f1b3dc5..1321b705469 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTConfigFormBuilder.tsx @@ -69,6 +69,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ dbtClassificationName: data.dbtClassificationName, dbtUpdateDescriptions: data.dbtUpdateDescriptions, includeTags: data.includeTags, + parsingTimeoutLimit: data.parsingTimeoutLimit, }, databaseFilterPattern: data.databaseFilterPattern, schemaFilterPattern: data.schemaFilterPattern, @@ -107,6 +108,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ dbtUpdateDescriptions={dbtConfigSource?.dbtUpdateDescriptions} enableDebugLog={data.enableDebugLog} includeTags={dbtConfigSource?.includeTags} + parsingTimeoutLimit={dbtConfigSource?.parsingTimeoutLimit} /> ); } @@ -120,6 +122,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ dbtUpdateDescriptions={dbtConfigSource?.dbtUpdateDescriptions} enableDebugLog={data.enableDebugLog} includeTags={dbtConfigSource?.includeTags} + parsingTimeoutLimit={dbtConfigSource?.parsingTimeoutLimit} /> ); } @@ -133,6 +136,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ dbtUpdateDescriptions={dbtConfigSource?.dbtUpdateDescriptions} enableDebugLog={data.enableDebugLog} includeTags={dbtConfigSource?.includeTags} + parsingTimeoutLimit={dbtConfigSource?.parsingTimeoutLimit} /> ); } @@ -145,6 +149,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ dbtUpdateDescriptions={dbtConfigSource?.dbtUpdateDescriptions} enableDebugLog={data.enableDebugLog} includeTags={dbtConfigSource?.includeTags} + parsingTimeoutLimit={dbtConfigSource?.parsingTimeoutLimit} /> ); } @@ -158,6 +163,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ enableDebugLog={data.enableDebugLog} gcsType={gcsConfigType} includeTags={dbtConfigSource?.includeTags} + parsingTimeoutLimit={dbtConfigSource?.parsingTimeoutLimit} /> ); } @@ -295,6 +301,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ }, ingestionName: value?.name, enableDebugLog: value?.loggerLevel, + parsingTimeoutLimit: value?.parsingTimeoutLimit, }); onSubmit(); } @@ -315,6 +322,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ }, ingestionName: value?.name, enableDebugLog: value?.loggerLevel, + parsingTimeoutLimit: value?.parsingTimeoutLimit, }); onSubmit(); } @@ -336,6 +344,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ }, ingestionName: value?.name, enableDebugLog: value?.loggerLevel, + parsingTimeoutLimit: value?.parsingTimeoutLimit, }); onSubmit(); } @@ -367,6 +376,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ }, ingestionName: value?.name, enableDebugLog: value?.loggerLevel, + parsingTimeoutLimit: value?.parsingTimeoutLimit, }); onSubmit(); } @@ -430,6 +440,7 @@ const DBTConfigFormBuilder: FunctionComponent = ({ }, ingestionName: value?.name, enableDebugLog: value?.loggerLevel, + parsingTimeoutLimit: value?.parsingTimeoutLimit, }); onSubmit(); } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTFormConstants.ts b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTFormConstants.ts index 59ccd09fb4c..9275b5a9b6f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTFormConstants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTFormConstants.ts @@ -95,3 +95,5 @@ export const rulesDBTGCSCredsFields: Record< email: ['clientEmail'], url: ['authUri', 'tokenUri', 'authProviderX509CertUrl', 'clientX509CertUrl'], }; + +export const dbtParsingTimeoutLimit = 300; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.test.tsx index bf3b033e564..0498be94add 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.test.tsx @@ -17,6 +17,7 @@ import { DBTBucketDetails, } from 'generated/metadataIngestion/dbtPipeline'; import React from 'react'; +import { dbtParsingTimeoutLimit } from './DBTFormConstants'; import { GCS_CONFIG } from './DBTFormEnum'; import { DBTGCSConfig } from './DBTGCSConfig'; @@ -27,6 +28,7 @@ const mockProps = { dbtClassificationName: '', dbtSecurityConfig: {} as Credentials, dbtPrefixConfig: {} as DBTBucketDetails, + parsingTimeoutLimit: dbtParsingTimeoutLimit, }; jest.mock('./DBTCommonFields.component', () => diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.tsx index a800db6af67..f92b4d8a48b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.tsx @@ -24,6 +24,7 @@ import { GCS_CONFIG } from './DBTFormEnum'; interface Props extends DbtConfigS3GCS { gcsType?: GCS_CONFIG; enableDebugLog: boolean; + parsingTimeoutLimit: number; } export const DBTGCSConfig: FunctionComponent = ({ @@ -34,6 +35,7 @@ export const DBTGCSConfig: FunctionComponent = ({ includeTags = true, dbtClassificationName, enableDebugLog, + parsingTimeoutLimit, }: Props) => { const dbtPrefixConfigFields: FieldProp[] = [ { @@ -252,6 +254,7 @@ export const DBTGCSConfig: FunctionComponent = ({ descriptionId="gcs-update-description" enableDebugLog={enableDebugLog} includeTags={includeTags} + parsingTimeoutLimit={parsingTimeoutLimit} /> ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTHttpConfig.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTHttpConfig.test.tsx index e061c0f51f0..53436b72401 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTHttpConfig.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTHttpConfig.test.tsx @@ -13,6 +13,7 @@ import { act, fireEvent, render, screen } from '@testing-library/react'; import React from 'react'; +import { dbtParsingTimeoutLimit } from './DBTFormConstants'; import { DBTHttpConfig } from './DBTHttpConfig'; jest.mock('./DBTCommonFields.component', () => @@ -25,6 +26,7 @@ const mockProps = { dbtRunResultsHttpPath: '', dbtUpdateDescriptions: false, enableDebugLog: false, + parsingTimeoutLimit: dbtParsingTimeoutLimit, }; describe('Test DBT Http Config Form', () => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTHttpConfig.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTHttpConfig.tsx index 3055c32df81..fda060a9ef6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTHttpConfig.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTHttpConfig.tsx @@ -20,6 +20,7 @@ import { DbtConfigHttp } from './DBTConfigForm.interface'; interface Props extends DbtConfigHttp { enableDebugLog: boolean; + parsingTimeoutLimit: number; } export const DBTHttpConfig: FunctionComponent = ({ @@ -30,6 +31,7 @@ export const DBTHttpConfig: FunctionComponent = ({ includeTags = true, dbtClassificationName, enableDebugLog, + parsingTimeoutLimit, }: Props) => { const { t } = useTranslation(); @@ -85,6 +87,7 @@ export const DBTHttpConfig: FunctionComponent = ({ descriptionId="http-update-description" enableDebugLog={enableDebugLog} includeTags={includeTags} + parsingTimeoutLimit={parsingTimeoutLimit} /> ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTLocalConfig.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTLocalConfig.test.tsx index 282e5e85cc8..38368e20634 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTLocalConfig.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTLocalConfig.test.tsx @@ -13,6 +13,7 @@ import { act, fireEvent, render, screen } from '@testing-library/react'; import React from 'react'; +import { dbtParsingTimeoutLimit } from './DBTFormConstants'; import { DBTLocalConfig } from './DBTLocalConfig'; const mockProps = { @@ -21,6 +22,7 @@ const mockProps = { dbtRunResultsFilePath: '', dbtUpdateDescriptions: false, enableDebugLog: false, + parsingTimeoutLimit: dbtParsingTimeoutLimit, }; jest.mock('./DBTCommonFields.component', () => diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTLocalConfig.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTLocalConfig.tsx index 746bf9c638a..a05ca952d30 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTLocalConfig.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTLocalConfig.tsx @@ -20,6 +20,7 @@ import { DbtConfigLocal } from './DBTConfigForm.interface'; interface Props extends DbtConfigLocal { enableDebugLog: boolean; + parsingTimeoutLimit: number; } export const DBTLocalConfig: FunctionComponent = ({ @@ -30,6 +31,7 @@ export const DBTLocalConfig: FunctionComponent = ({ includeTags = true, dbtClassificationName, enableDebugLog, + parsingTimeoutLimit, }: Props) => { const localConfigFields: FieldProp[] = [ { @@ -85,6 +87,7 @@ export const DBTLocalConfig: FunctionComponent = ({ descriptionId="local-update-description" enableDebugLog={enableDebugLog} includeTags={includeTags} + parsingTimeoutLimit={parsingTimeoutLimit} /> ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTS3Config.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTS3Config.test.tsx index 7ab05c36e89..c6ad34e6084 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTS3Config.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTS3Config.test.tsx @@ -13,10 +13,12 @@ import { render, screen } from '@testing-library/react'; import React from 'react'; +import { dbtParsingTimeoutLimit } from './DBTFormConstants'; import { DBTS3Config } from './DBTS3Config'; const mockProps = { enableDebugLog: false, + parsingTimeoutLimit: dbtParsingTimeoutLimit, }; jest.mock('./DBTCommonFields.component', () => diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTS3Config.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTS3Config.tsx index 72ee02690d7..154ff2152e6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTS3Config.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTS3Config.tsx @@ -20,6 +20,7 @@ import { DbtConfigS3GCS } from './DBTConfigForm.interface'; interface Props extends DbtConfigS3GCS { enableDebugLog: boolean; + parsingTimeoutLimit: number; } export const DBTS3Config: FunctionComponent = ({ @@ -29,6 +30,7 @@ export const DBTS3Config: FunctionComponent = ({ includeTags = true, dbtClassificationName, enableDebugLog, + parsingTimeoutLimit, }: Props) => { const s3ConfigFields: FieldProp[] = [ { @@ -187,6 +189,7 @@ export const DBTS3Config: FunctionComponent = ({ descriptionId="s3-update-description" enableDebugLog={enableDebugLog} includeTags={includeTags} + parsingTimeoutLimit={parsingTimeoutLimit} /> ); diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Ingestions.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Ingestions.constant.ts index 342ee6cd8e8..613bfe2322f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/Ingestions.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/Ingestions.constant.ts @@ -59,3 +59,5 @@ export const PIPELINE_TYPE_LOCALIZATION = { }; export const DBT_CLASSIFICATION_DEFAULT_VALUE = 'dbtTags'; + +export const DEFAULT_PARSING_TIMEOUT_LIMIT = 300; diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json index 1c15ed2798e..a6676bb3ee1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json @@ -378,6 +378,7 @@ "filter": "Filter", "filter-pattern": "Filter Pattern", "filter-plural": "Filters", + "filtering-condition": "Filtering Condition", "first": "First", "first-lowercase": "first", "first-quartile": "First Quartile", @@ -643,6 +644,7 @@ "page-views-by-data-asset-plural": "Page Views by Data Assets", "parameter": "Parameter", "parent": "Parent", + "parsing-timeout-limit": "Query Parsing Timeout Limit", "partition-lowercase-plural": "partitions", "partition-plural": "Partitions", "partitioned": "Partitioned", @@ -1021,6 +1023,7 @@ "view-entity": "View {{entity}}", "view-more": "View more", "view-new-count": "View {{count}} new", + "view-parsing-timeout-limit": "View Definition Parsing Timeout Limit", "view-plural": "Views", "volume-change": "Volume Change", "warning": "Warning", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json index a1f961efa24..f293d68741c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json @@ -378,6 +378,7 @@ "filter": "Filtro", "filter-pattern": "Patrón de Filtro", "filter-plural": "Filtros", + "filtering-condition": "Filtering Condition", "first": "Primero", "first-lowercase": "primero", "first-quartile": "Primer Cuartil", @@ -643,6 +644,7 @@ "page-views-by-data-asset-plural": "Vistas de página por activos de datos", "parameter": "Parámetro", "parent": "Padre", + "parsing-timeout-limit": "Query Parsing Timeout Limit", "partition-lowercase-plural": "particiones", "partition-plural": "Particiones", "partitioned": "Partitioned", @@ -1021,6 +1023,7 @@ "view-entity": "Ver {{entity}}", "view-more": "Ver más", "view-new-count": "Ver {{count}} nuevo", + "view-parsing-timeout-limit": "View Definition Parsing Timeout Limit", "view-plural": "Vistas", "volume-change": "Volume Change", "warning": "Warning", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json index 06e1f4c5a4a..b7a86e2d9f8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json @@ -378,6 +378,7 @@ "filter": "Filtre", "filter-pattern": "Configuration du Filtre", "filter-plural": "Filtres", + "filtering-condition": "Filtering Condition", "first": "Premier", "first-lowercase": "premier", "first-quartile": "Premier Quartile", @@ -643,6 +644,7 @@ "page-views-by-data-asset-plural": "Page vues par actif de données", "parameter": "Paramètre", "parent": "Parent", + "parsing-timeout-limit": "Query Parsing Timeout Limit", "partition-lowercase-plural": "partitions", "partition-plural": "Partitions", "partitioned": "Partitionné", @@ -1021,6 +1023,7 @@ "view-entity": "Voir {{entity}}", "view-more": "Voir plus", "view-new-count": "Voir {{count}} nouveau", + "view-parsing-timeout-limit": "View Definition Parsing Timeout Limit", "view-plural": "Voir", "volume-change": "Volume Change", "warning": "Attention", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json index 51d4212571c..24f418afcac 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json @@ -378,6 +378,7 @@ "filter": "Filter", "filter-pattern": "フィルターのパターン", "filter-plural": "フィルター", + "filtering-condition": "Filtering Condition", "first": "最初", "first-lowercase": "最初", "first-quartile": "第1四分位数", @@ -643,6 +644,7 @@ "page-views-by-data-asset-plural": "データアセットごとのページビュー", "parameter": "パラメータ", "parent": "親", + "parsing-timeout-limit": "Query Parsing Timeout Limit", "partition-lowercase-plural": "パーティション", "partition-plural": "パーティション", "partitioned": "Partitioned", @@ -1021,6 +1023,7 @@ "view-entity": "{{entity}}を見る", "view-more": "もっと見る", "view-new-count": "View {{count}} new", + "view-parsing-timeout-limit": "View Definition Parsing Timeout Limit", "view-plural": "Views", "volume-change": "Volume Change", "warning": "Warning", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json index 21d8f0db402..99b1708ac58 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json @@ -378,6 +378,7 @@ "filter": "Filter", "filter-pattern": "Padrão de filtro", "filter-plural": "Filtros", + "filtering-condition": "Filtering Condition", "first": "Primeiro", "first-lowercase": "primeiro", "first-quartile": "Primeiro quartil", @@ -643,6 +644,7 @@ "page-views-by-data-asset-plural": "Visualizações da página por ativo de dados", "parameter": "Parâmetro", "parent": "Pai", + "parsing-timeout-limit": "Query Parsing Timeout Limit", "partition-lowercase-plural": "partições", "partition-plural": "Partições", "partitioned": "Partitioned", @@ -1021,6 +1023,7 @@ "view-entity": "Ver {{entity}}", "view-more": "Ver mais", "view-new-count": "Ver {{count}} novo", + "view-parsing-timeout-limit": "View Definition Parsing Timeout Limit", "view-plural": "Visualizações", "volume-change": "Volume Change", "warning": "Warning", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json index 53e90a501d0..ebb67e4aaa7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json @@ -378,6 +378,7 @@ "filter": "过滤", "filter-pattern": "过滤条件", "filter-plural": "过滤", + "filtering-condition": "Filtering Condition", "first": "第一", "first-lowercase": "第一", "first-quartile": "第一四分位数", @@ -643,6 +644,7 @@ "page-views-by-data-asset-plural": "数据资产页面浏览量", "parameter": "参数", "parent": "父级", + "parsing-timeout-limit": "Query Parsing Timeout Limit", "partition-lowercase-plural": "分区", "partition-plural": "分区", "partitioned": "已分区", @@ -1021,6 +1023,7 @@ "view-entity": "查看{{entity}}", "view-more": "查看更多", "view-new-count": "查看{{count}}个新的", + "view-parsing-timeout-limit": "View Definition Parsing Timeout Limit", "view-plural": "查看", "volume-change": "Volume Change", "warning": "警告",