diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery.py b/ingestion/src/metadata/ingestion/source/database/bigquery.py index b5ce961b38a..532628849ba 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery.py @@ -122,6 +122,8 @@ class BigquerySource(CommonDbSourceService): :param _: :return: """ + if not self.source_config.includeTags: + return taxonomies = PolicyTagManagerClient().list_taxonomies( parent=f"projects/{self.project_id}/locations/{self.service_connection.taxonomyLocation}" ) @@ -155,6 +157,8 @@ class BigquerySource(CommonDbSourceService): This will only get executed if the tags context is properly informed """ + if not self.source_config.includeTags: + return if column.get("policy_tags"): return [ TagLabel( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx index 7530196c43a..b5c45ed5101 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx @@ -161,6 +161,9 @@ const AddIngestion = ({ const [includeView, setIncludeView] = useState( Boolean((data?.sourceConfig.config as ConfigClass)?.includeViews) ); + const [includeTag, setIncludeTags] = useState( + Boolean((data?.sourceConfig.config as ConfigClass)?.includeTags) + ); const [includeLineage, setIncludeLineage] = useState( Boolean((data?.sourceConfig.config as ConfigClass)?.includeLineage ?? true) ); @@ -391,6 +394,7 @@ const AddIngestion = ({ return { includeViews: includeView, + includeTags: includeTag, databaseFilterPattern: getFilterPatternData( databaseFilterPattern, showDatabaseFilter @@ -618,6 +622,7 @@ const AddIngestion = ({ handleDescription={(val) => setDescription(val)} handleEnableDebugLog={() => setEnableDebugLog((pre) => !pre)} handleIncludeLineage={() => setIncludeLineage((pre) => !pre)} + handleIncludeTags={() => setIncludeTags((pre) => !pre)} handleIncludeView={() => setIncludeView((pre) => !pre)} handleIngestSampleData={() => setIngestSampleData((pre) => !pre)} handleIngestionName={(val) => setIngestionName(val)} @@ -627,6 +632,7 @@ const AddIngestion = ({ handleShowFilter={handleShowFilter} handleStageFileLocation={(val) => setStageFileLocation(val)} includeLineage={includeLineage} + includeTags={includeTag} includeView={includeView} ingestSampleData={ingestSampleData} ingestionName={ingestionName} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx index ffab2650822..2b404978cfe 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx @@ -64,6 +64,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = { }, includeLineage: false, includeView: false, + includeTags: false, pipelineType: PipelineType.Metadata, queryLogDuration: 1, resultLimit: 100, @@ -79,6 +80,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = { showFqnFilter: false, handleIncludeLineage: jest.fn(), handleIncludeView: jest.fn(), + handleIncludeTags: jest.fn(), handleIngestionName: jest.fn(), handleMarkDeletedTables: jest.fn(), handleQueryLogDuration: jest.fn(), @@ -124,6 +126,6 @@ describe('Test ConfigureIngestion component', () => { expect(backButton).toBeInTheDocument(); expect(nextButton).toBeInTheDocument(); expect(filterPatternComponents.length).toBe(3); - expect(toggleSwitchs.length).toBe(3); + expect(toggleSwitchs.length).toBe(4); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx index 8a540fd6032..94b4f8b20b8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx @@ -39,6 +39,7 @@ const ConfigureIngestion = ({ fqnFilterPattern, includeLineage, includeView, + includeTags, markDeletedTables, serviceCategory, pipelineType, @@ -63,6 +64,7 @@ const ConfigureIngestion = ({ handleShowFilter, handleIncludeLineage, handleIncludeView, + handleIncludeTags, handleMarkDeletedTables, handleIngestSampleData, handleDatasetServiceName, @@ -128,6 +130,20 @@ const ConfigureIngestion = ({
{getSeparator('')} ++ Enable extracting tags from the data source +
+ {getSeparator('')} +