diff --git a/catalog-rest-service/src/main/resources/json/schema/metadataIngestion/databaseServiceProfilerPipeline.json b/catalog-rest-service/src/main/resources/json/schema/metadataIngestion/databaseServiceProfilerPipeline.json index e572d9ee2a2..5be5c2b354c 100644 --- a/catalog-rest-service/src/main/resources/json/schema/metadataIngestion/databaseServiceProfilerPipeline.json +++ b/catalog-rest-service/src/main/resources/json/schema/metadataIngestion/databaseServiceProfilerPipeline.json @@ -17,8 +17,16 @@ "$ref": "#/definitions/profilerConfigType", "default": "Profiler" }, - "fqnFilterPattern": { - "description": "Regex to only fetch tables with FQN matching the pattern.", + "schemaFilterPattern": { + "description": "Regex to only fetch tables or databases that matches the pattern.", + "$ref": "../type/filterPattern.json#/definitions/filterPattern" + }, + "tableFilterPattern": { + "description": "Regex exclude tables or databases that matches the pattern.", + "$ref": "../type/filterPattern.json#/definitions/filterPattern" + }, + "databaseFilterPattern": { + "description": "Regex to only fetch databases that matches the pattern.", "$ref": "../type/filterPattern.json#/definitions/filterPattern" }, "generateSampleData": { diff --git a/docker/local-metadata/docker-compose.yml b/docker/local-metadata/docker-compose.yml index 544af38e91a..a6a3355fb55 100644 --- a/docker/local-metadata/docker-compose.yml +++ b/docker/local-metadata/docker-compose.yml @@ -122,7 +122,7 @@ services: ingestion: build: context: ../../. - dockerfile: ingestion/Dockerfile + dockerfile: ingestion/Dockerfile_local args: INGESTION_DEPENDENCY: ${INGESTION_DEPENDENCY:-all} container_name: openmetadata_ingestion diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index f9ad38cdb74..3162d4d8d7d 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -25,13 +25,12 @@ COPY openmetadata-airflow-apis /openmetadata-airflow-apis RUN pip install "." - FROM apis as ingestion WORKDIR /ingestion COPY ingestion /ingestion ARG INGESTION_DEPENDENCY=all -RUN pip install ".[${INGESTION_DEPENDENCY}]" +RUN pip install --upgrade ".[${INGESTION_DEPENDENCY}]" RUN airflow db init RUN cp -r /ingestion/airflow.cfg /airflow/airflow.cfg diff --git a/ingestion/Dockerfile_local b/ingestion/Dockerfile_local new file mode 100644 index 00000000000..355eac2c43b --- /dev/null +++ b/ingestion/Dockerfile_local @@ -0,0 +1,40 @@ +FROM python:3.9-slim as base +ENV AIRFLOW_HOME=/airflow +RUN apt-get update && \ + apt-get install -y gcc libsasl2-modules libsasl2-dev build-essential libssl-dev libffi-dev librdkafka-dev unixodbc-dev python3.9-dev openjdk-11-jre unixodbc freetds-dev freetds-bin tdsodbc libevent-dev wget openssl --no-install-recommends && \ + rm -rf /var/lib/apt/lists/* + +# Manually fix security vulnerability from curl +# - https://security.snyk.io/vuln/SNYK-DEBIAN11-CURL-2936229 +# Add it back to the usual apt-get install once a fix for Debian is released +RUN wget https://curl.se/download/curl-7.84.0.tar.gz && \ + tar -xvf curl-7.84.0.tar.gz && cd curl-7.84.0 && \ + ./configure --with-openssl && make && make install + + +FROM base as airflow +ENV AIRFLOW_VERSION=2.3.3 +ENV CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.9.txt" +# Add docker provider for the DockerOperator +RUN pip install "apache-airflow[docker]==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}" + + +FROM airflow as apis +WORKDIR /openmetadata-airflow-apis +COPY openmetadata-airflow-apis /openmetadata-airflow-apis + +RUN pip install "." "openmetadata-ingestion[all]" +RUN pip uninstall openmetadata-ingestion -y + +FROM apis as ingestion +WORKDIR /ingestion +COPY ingestion /ingestion + +ARG INGESTION_DEPENDENCY=all +RUN pip install --upgrade ".[${INGESTION_DEPENDENCY}]" + +RUN airflow db init +RUN cp -r /ingestion/airflow.cfg /airflow/airflow.cfg +RUN chmod 755 ingestion_dependency.sh +EXPOSE 8080 +CMD [ "./ingestion_dependency.sh" ] diff --git a/ingestion/examples/workflows/mysql_profiler.yaml b/ingestion/examples/workflows/mysql_profiler.yaml index d209706e3ef..861527cfb4e 100644 --- a/ingestion/examples/workflows/mysql_profiler.yaml +++ b/ingestion/examples/workflows/mysql_profiler.yaml @@ -13,9 +13,9 @@ source: config: type: Profiler generateSampleData: true - fqnFilterPattern: + schemaFilterPattern: includes: - - local_mysql.openmetadata_db* + - openmetadata_db* processor: type: "orm-profiler" diff --git a/ingestion/src/metadata/orm_profiler/api/workflow.py b/ingestion/src/metadata/orm_profiler/api/workflow.py index de2b4033a82..69506adec35 100644 --- a/ingestion/src/metadata/orm_profiler/api/workflow.py +++ b/ingestion/src/metadata/orm_profiler/api/workflow.py @@ -16,6 +16,7 @@ Workflow definition for the ORM Profiler. - How to specify the entities to run - How to define metrics & tests """ +import traceback from copy import deepcopy from typing import Iterable, List @@ -50,7 +51,7 @@ from metadata.utils.class_helper import ( get_service_type_from_source_type, ) from metadata.utils.connections import get_connection, test_connection -from metadata.utils.filters import filter_by_fqn +from metadata.utils.filters import filter_by_fqn, filter_by_schema, filter_by_table from metadata.utils.logger import profiler_logger logger = profiler_logger() @@ -123,18 +124,31 @@ class ProfilerWorkflow: We will update the status on the SQLSource Status. """ for table in tables: + try: + if filter_by_schema( + self.source_config.schemaFilterPattern, + table.databaseSchema.name, + ): + self.source_status.filter( + table.databaseSchema.fullyQualifiedName, + "Schema pattern not allowed", + ) + continue + if filter_by_table( + self.source_config.tableFilterPattern, + table.name.__root__, + ): + self.source_status.filter( + table.fullyQualifiedName.__root__, "Table pattern not allowed" + ) + continue - if filter_by_fqn( - fqn_filter_pattern=self.source_config.fqnFilterPattern, - fqn=table.fullyQualifiedName.__root__, - ): - self.source_status.filter( - table.fullyQualifiedName.__root__, "Schema pattern not allowed" - ) - continue - - self.source_status.scanned(table.fullyQualifiedName.__root__) - yield table + self.source_status.scanned(table.fullyQualifiedName.__root__) + yield table + except Exception as err: # pylint: disable=broad-except + self.source_status.filter(table.fullyQualifiedName.__root__, f"{err}") + logger.error(err) + logger.debug(traceback.format_exc()) def create_processor(self, service_connection_config): self.processor_interface: InterfaceProtocol = SQAProfilerInterface( @@ -201,38 +215,48 @@ class ProfilerWorkflow: yield from self.filter_entities(all_tables) + def copy_service_config(self, database) -> None: + copy_service_connection_config = deepcopy( + self.config.source.serviceConnection.__root__.config + ) + if hasattr( + self.config.source.serviceConnection.__root__.config, + "supportsDatabase", + ): + if hasattr( + self.config.source.serviceConnection.__root__.config, "database" + ): + copy_service_connection_config.database = database.name.__root__ + if hasattr(self.config.source.serviceConnection.__root__.config, "catalog"): + copy_service_connection_config.catalog = database.name.__root__ + + self.create_processor(copy_service_connection_config) + def execute(self): """ Run the profiling and tests """ - copy_service_connection_config = deepcopy( - self.config.source.serviceConnection.__root__.config - ) + for database in self.get_database_entities(): - if hasattr( - self.config.source.serviceConnection.__root__.config, "supportsDatabase" - ): - if hasattr( - self.config.source.serviceConnection.__root__.config, "database" - ): - copy_service_connection_config.database = database.name.__root__ - if hasattr( - self.config.source.serviceConnection.__root__.config, "catalog" - ): - copy_service_connection_config.catalog = database.name.__root__ + try: + self.copy_service_config(database) - self.create_processor(copy_service_connection_config) + for entity in self.get_table_entities(database=database): + try: + profile_and_tests: ProfilerResponse = self.processor.process( + record=entity, + generate_sample_data=self.source_config.generateSampleData, + ) - for entity in self.get_table_entities(database=database): - profile_and_tests: ProfilerResponse = self.processor.process( - record=entity, - generate_sample_data=self.source_config.generateSampleData, - ) - - if hasattr(self, "sink"): - self.sink.write_record(profile_and_tests) - - self.processor_interface.session.close() + if hasattr(self, "sink"): + self.sink.write_record(profile_and_tests) + except Exception as err: # pylint: disable=broad-except + logger.error(err) + logger.debug(traceback.format_exc()) + self.processor_interface.session.close() + except Exception as err: # pylint: disable=broad-except + logger.error(err) + logger.debug(traceback.format_exc()) def print_status(self) -> int: """ diff --git a/ingestion/tests/integration/orm_profiler/test_orm_profiler.py b/ingestion/tests/integration/orm_profiler/test_orm_profiler.py index 661c79aaa9f..041b3032914 100644 --- a/ingestion/tests/integration/orm_profiler/test_orm_profiler.py +++ b/ingestion/tests/integration/orm_profiler/test_orm_profiler.py @@ -163,7 +163,7 @@ class ProfilerWorkflowTest(TestCase): # workflow_config["source"]["sourceConfig"]["config"].update( # { # "type": "Profiler", - # "fqnFilterPattern": {"includes": ["test_sqlite.main.main.users"]}, + # "tableFilterPattern": {"includes": ["users"]}, # } # ) # workflow_config["processor"] = { @@ -233,7 +233,7 @@ class ProfilerWorkflowTest(TestCase): { "type": "Profiler", "profileSample": 50, - "fqnFilterPattern": {"includes": ["test_sqlite.main.main.new_users"]}, + "tableFilterPattern": {"includes": ["new_users"]}, } ) workflow_config["processor"] = {"type": "orm-profiler", "config": {}} diff --git a/ingestion/tests/unit/profiler/test_workflow.py b/ingestion/tests/unit/profiler/test_workflow.py index 883df17a66f..4961a4279e8 100644 --- a/ingestion/tests/unit/profiler/test_workflow.py +++ b/ingestion/tests/unit/profiler/test_workflow.py @@ -128,17 +128,17 @@ def test_filter_entities(mocked_method): # We can exclude based on the schema name exclude_config = deepcopy(config) - exclude_config["source"]["sourceConfig"]["config"]["fqnFilterPattern"] = { - "excludes": ["service*"] + exclude_config["source"]["sourceConfig"]["config"]["schemaFilterPattern"] = { + "excludes": ["another_schema"] } exclude_workflow = ProfilerWorkflow.create(exclude_config) mocked_method.assert_called() - assert len(list(exclude_workflow.filter_entities(all_tables))) == 0 + assert len(list(exclude_workflow.filter_entities(all_tables))) == 2 exclude_config = deepcopy(config) - exclude_config["source"]["sourceConfig"]["config"]["fqnFilterPattern"] = { - "excludes": ["service.db.another*"] + exclude_config["source"]["sourceConfig"]["config"]["schemaFilterPattern"] = { + "excludes": ["another*"] } exclude_workflow = ProfilerWorkflow.create(exclude_config) @@ -146,8 +146,8 @@ def test_filter_entities(mocked_method): assert len(list(exclude_workflow.filter_entities(all_tables))) == 2 include_config = deepcopy(config) - include_config["source"]["sourceConfig"]["config"]["fqnFilterPattern"] = { - "includes": ["service*"] + include_config["source"]["sourceConfig"]["config"]["databaseFilterPattern"] = { + "includes": ["db*"] } include_workflow = ProfilerWorkflow.create(include_config) diff --git a/openmetadata-core/src/main/resources/json/schema/metadataIngestion/databaseServiceProfilerPipeline.json b/openmetadata-core/src/main/resources/json/schema/metadataIngestion/databaseServiceProfilerPipeline.json index 559fcf36b08..76091f0c937 100644 --- a/openmetadata-core/src/main/resources/json/schema/metadataIngestion/databaseServiceProfilerPipeline.json +++ b/openmetadata-core/src/main/resources/json/schema/metadataIngestion/databaseServiceProfilerPipeline.json @@ -17,8 +17,16 @@ "$ref": "#/definitions/profilerConfigType", "default": "Profiler" }, - "fqnFilterPattern": { - "description": "Regex to only fetch tables with FQN matching the pattern.", + "schemaFilterPattern": { + "description": "Regex to only fetch tables or databases that matches the pattern.", + "$ref": "../type/filterPattern.json#/definitions/filterPattern" + }, + "tableFilterPattern": { + "description": "Regex exclude tables or databases that matches the pattern.", + "$ref": "../type/filterPattern.json#/definitions/filterPattern" + }, + "databaseFilterPattern": { + "description": "Regex to only fetch databases that matches the pattern.", "$ref": "../type/filterPattern.json#/definitions/filterPattern" }, "generateSampleData": { diff --git a/openmetadata-docs/content/main-concepts/metadata-standard/schemas/metadataIngestion/databaseServiceProfilerPipeline.md b/openmetadata-docs/content/main-concepts/metadata-standard/schemas/metadataIngestion/databaseServiceProfilerPipeline.md index fbedb3e9cb1..7508f372662 100644 --- a/openmetadata-docs/content/main-concepts/metadata-standard/schemas/metadataIngestion/databaseServiceProfilerPipeline.md +++ b/openmetadata-docs/content/main-concepts/metadata-standard/schemas/metadataIngestion/databaseServiceProfilerPipeline.md @@ -10,7 +10,9 @@ slug: /main-concepts/metadata-standard/schemas/metadataingestion/databaseservice ## Properties - **`type`**: Pipeline type. Refer to *#/definitions/profilerConfigType*. Default: `Profiler`. -- **`fqnFilterPattern`**: Regex to only fetch tables with FQN matching the pattern. Refer to *../type/filterPattern.json#/definitions/filterPattern*. +- **`databaseFilterPattern`**: Regex to only fetch database with database name matching the pattern. Refer to *../type/filterPattern.json#/definitions/filterPattern*. +- **`schemaFilterPattern`**: Regex to only fetch schema with schema name matching the pattern. Refer to *../type/filterPattern.json#/definitions/filterPattern*. +- **`tableFilterPattern`**: Regex to only fetch tables with table name matching the pattern. Refer to *../type/filterPattern.json#/definitions/filterPattern*. - **`generateSampleData`** *(boolean)*: Option to turn on/off generating sample data. Default: `True`. ## Definitions diff --git a/openmetadata-docs/ingestion/connectors/athena/profiler.yaml b/openmetadata-docs/ingestion/connectors/athena/profiler.yaml index 1679dad73ef..40e4db30d57 100644 --- a/openmetadata-docs/ingestion/connectors/athena/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/athena/profiler.yaml @@ -13,7 +13,9 @@ source: sourceConfig: config: type: Profiler - fqnFilterPattern: + databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-docs/ingestion/connectors/azuresql/profiler.yaml b/openmetadata-docs/ingestion/connectors/azuresql/profiler.yaml index 992c6c44a09..b38a47c3068 100644 --- a/openmetadata-docs/ingestion/connectors/azuresql/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/azuresql/profiler.yaml @@ -12,7 +12,9 @@ source: sourceConfig: config: type: Profiler - fqnFilterPattern:
+ databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-docs/ingestion/connectors/bigquery/profiler.yaml b/openmetadata-docs/ingestion/connectors/bigquery/profiler.yaml index c663fee4fcf..1ee038869ec 100644 --- a/openmetadata-docs/ingestion/connectors/bigquery/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/bigquery/profiler.yaml @@ -18,7 +18,9 @@ source: clientX509CertUrl: sourceConfig: config: - fqnFilterPattern:
+ databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-docs/ingestion/connectors/db2/profiler.yaml b/openmetadata-docs/ingestion/connectors/db2/profiler.yaml index 55190580dba..d5e234c0bb0 100644 --- a/openmetadata-docs/ingestion/connectors/db2/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/db2/profiler.yaml @@ -10,7 +10,9 @@ source: sourceConfig: config: type: Profiler - fqnFilterPattern:
+ databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-docs/ingestion/connectors/hive/profiler.yaml b/openmetadata-docs/ingestion/connectors/hive/profiler.yaml index e6d94613785..90b5d777714 100644 --- a/openmetadata-docs/ingestion/connectors/hive/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/hive/profiler.yaml @@ -11,7 +11,9 @@ source: sourceConfig: config: type: Profiler - fqnFilterPattern:
+ databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-docs/ingestion/connectors/mariadb/profiler.yaml b/openmetadata-docs/ingestion/connectors/mariadb/profiler.yaml index 7c9570b61d7..905f6ef5d8e 100644 --- a/openmetadata-docs/ingestion/connectors/mariadb/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/mariadb/profiler.yaml @@ -10,7 +10,9 @@ source: sourceConfig: config: type: Profiler - fqnFilterPattern:
+ databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-docs/ingestion/connectors/postgres/profiler.yaml b/openmetadata-docs/ingestion/connectors/postgres/profiler.yaml index 71fba648fc7..8474ef1fddf 100644 --- a/openmetadata-docs/ingestion/connectors/postgres/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/postgres/profiler.yaml @@ -10,7 +10,9 @@ source: sourceConfig: config: type: Profiler - fqnFilterPattern:
+ databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-docs/ingestion/connectors/presto/profiler.yaml b/openmetadata-docs/ingestion/connectors/presto/profiler.yaml index cc3da848b86..54de483969a 100644 --- a/openmetadata-docs/ingestion/connectors/presto/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/presto/profiler.yaml @@ -12,7 +12,9 @@ source: sourceConfig: config: type: Profiler - fqnFilterPattern:
+ databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-docs/ingestion/connectors/redshift/profiler.yaml b/openmetadata-docs/ingestion/connectors/redshift/profiler.yaml index cd0880a9ddf..e2df3014214 100644 --- a/openmetadata-docs/ingestion/connectors/redshift/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/redshift/profiler.yaml @@ -11,7 +11,9 @@ source: sourceConfig: config: type: Profiler - fqnFilterPattern:
+ databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-docs/ingestion/connectors/salesforce/profiler.yaml b/openmetadata-docs/ingestion/connectors/salesforce/profiler.yaml index fb0e573e68d..25fe5c3cfd8 100644 --- a/openmetadata-docs/ingestion/connectors/salesforce/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/salesforce/profiler.yaml @@ -12,7 +12,9 @@ source: sourceConfig: config: type: Profiler - fqnFilterPattern:
+ databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-docs/ingestion/connectors/singlestore/profiler.yaml b/openmetadata-docs/ingestion/connectors/singlestore/profiler.yaml index 7b4eb11b749..1214d5ffdf9 100644 --- a/openmetadata-docs/ingestion/connectors/singlestore/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/singlestore/profiler.yaml @@ -10,7 +10,9 @@ source: sourceConfig: config: type: Profiler - fqnFilterPattern:
+ databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-docs/ingestion/connectors/trino/profiler.yaml b/openmetadata-docs/ingestion/connectors/trino/profiler.yaml index d0b40687707..516a34387a8 100644 --- a/openmetadata-docs/ingestion/connectors/trino/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/trino/profiler.yaml @@ -15,7 +15,9 @@ source: sourceConfig: config: type: Profiler - fqnFilterPattern:
+ databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-docs/ingestion/connectors/vertica/profiler.yaml b/openmetadata-docs/ingestion/connectors/vertica/profiler.yaml index fb0bb4f071c..d84d4876a84 100644 --- a/openmetadata-docs/ingestion/connectors/vertica/profiler.yaml +++ b/openmetadata-docs/ingestion/connectors/vertica/profiler.yaml @@ -10,7 +10,9 @@ source: sourceConfig: config: type: Profiler - fqnFilterPattern:
+ databaseFilterPattern:
+ schemaFilterPattern:
+ tableFilterPattern:
processor: type: orm-profiler config: {} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 225baf99c9d..9aa90204c56 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -127,9 +127,6 @@ const AddIngestion = ({ (data?.sourceConfig.config as ConfigClass)?.pipelineFilterPattern ) ); - const [showFqnFilter, setShowFqnFilter] = useState( - !isUndefined((data?.sourceConfig.config as ConfigClass)?.fqnFilterPattern) - ); const configData = useMemo( () => (data?.sourceConfig.config as DatabaseServiceMetadataPipelineClass) @@ -206,10 +203,6 @@ const AddIngestion = ({ (data?.sourceConfig.config as ConfigClass)?.pipelineFilterPattern ?? INITIAL_FILTER_PATTERN ); - const [fqnFilterPattern, setFqnFilterPattern] = useState( - (data?.sourceConfig.config as ConfigClass)?.fqnFilterPattern ?? - INITIAL_FILTER_PATTERN - ); const [queryLogDuration, setQueryLogDuration] = useState( (data?.sourceConfig.config as ConfigClass)?.queryLogDuration ?? 1 @@ -261,10 +254,6 @@ const AddIngestion = ({ case FilterPatternEnum.CHART: setChartFilterPattern({ ...chartFilterPattern, includes: value }); - break; - case FilterPatternEnum.FQN: - setFqnFilterPattern({ ...fqnFilterPattern, includes: value }); - break; case FilterPatternEnum.PIPELINE: setPipelineFilterPattern({ ...pipelineFilterPattern, includes: value }); @@ -300,10 +289,6 @@ const AddIngestion = ({ case FilterPatternEnum.CHART: setChartFilterPattern({ ...chartFilterPattern, excludes: value }); - break; - case FilterPatternEnum.FQN: - setFqnFilterPattern({ ...fqnFilterPattern, excludes: value }); - break; case FilterPatternEnum.PIPELINE: setPipelineFilterPattern({ ...pipelineFilterPattern, excludes: value }); @@ -337,10 +322,6 @@ const AddIngestion = ({ case FilterPatternEnum.CHART: setShowChartFilter(value); - break; - case FilterPatternEnum.FQN: - setShowFqnFilter(value); - break; case FilterPatternEnum.PIPELINE: setShowPipelineFilter(value); @@ -469,10 +450,19 @@ const AddIngestion = ({ } case PipelineType.Profiler: { return { - fqnFilterPattern: getFilterPatternData( - fqnFilterPattern, - showFqnFilter + databaseFilterPattern: getFilterPatternData( + databaseFilterPattern, + showDatabaseFilter ), + schemaFilterPattern: getFilterPatternData( + schemaFilterPattern, + showSchemaFilter + ), + tableFilterPattern: getFilterPatternData( + tableFilterPattern, + showTableFilter + ), + type: profilerIngestionType, generateSampleData: ingestSampleData, profileSample: profileSample, @@ -624,7 +614,6 @@ const AddIngestion = ({ databaseServiceName={databaseServiceName} description={description} enableDebugLog={enableDebugLog} - fqnFilterPattern={fqnFilterPattern} getExcludeValue={getExcludeValue} getIncludeValue={getIncludeValue} handleDatasetServiceName={(val) => setDatabaseServiceName(val)} @@ -658,7 +647,6 @@ const AddIngestion = ({ showChartFilter={showChartFilter} showDashboardFilter={showDashboardFilter} showDatabaseFilter={showDatabaseFilter} - showFqnFilter={showFqnFilter} showPipelineFilter={showPipelineFilter} showSchemaFilter={showSchemaFilter} showTableFilter={showTableFilter} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx index ab74e6c0a64..3c0be517947 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx @@ -58,10 +58,6 @@ const mockConfigureIngestion: ConfigureIngestionProps = { includes: [], excludes: [], }, - fqnFilterPattern: { - includes: [], - excludes: [], - }, includeLineage: false, includeView: false, includeTags: false, @@ -77,7 +73,6 @@ const mockConfigureIngestion: ConfigureIngestionProps = { showTopicFilter: false, showChartFilter: false, showPipelineFilter: false, - showFqnFilter: false, handleIncludeLineage: jest.fn(), handleIncludeView: jest.fn(), handleIncludeTags: jest.fn(), diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx index 0c0e9efc8ea..e4d03a4f805 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx @@ -36,7 +36,6 @@ const ConfigureIngestion = ({ topicFilterPattern, chartFilterPattern, pipelineFilterPattern, - fqnFilterPattern, includeLineage, includeView, includeTags, @@ -51,7 +50,6 @@ const ConfigureIngestion = ({ showTopicFilter, showChartFilter, showPipelineFilter, - showFqnFilter, queryLogDuration, stageFileLocation, threadCount, @@ -402,24 +400,6 @@ const ConfigureIngestion = ({ } }; - const getProfilerFilterPatternField = () => { - return ( - - - handleShowFilter(value, FilterPatternEnum.FQN) - } - includePattern={fqnFilterPattern?.includes ?? []} - type={FilterPatternEnum.FQN} - /> - - ); - }; - const getMetadataFields = () => { return ( <> @@ -521,7 +501,7 @@ const ConfigureIngestion = ({ {getSeparator('')} -
{getProfilerFilterPatternField()}
+
{getMetadataFilterPatternField()}
{getSeparator('')} {getProfileSample()} {getSeparator('')} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts index a0427acf244..2051d1e5b25 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts @@ -66,7 +66,6 @@ export interface ConfigureIngestionProps { topicFilterPattern: FilterPattern; chartFilterPattern: FilterPattern; pipelineFilterPattern: FilterPattern; - fqnFilterPattern: FilterPattern; includeLineage: boolean; includeView: boolean; includeTags: boolean; @@ -82,7 +81,6 @@ export interface ConfigureIngestionProps { showTopicFilter: boolean; showChartFilter: boolean; showPipelineFilter: boolean; - showFqnFilter: boolean; threadCount: number; queryLogDuration: number; stageFileLocation: string;