From dc5dfae3b8f14b613047676ba9536a9a87d79abb Mon Sep 17 00:00:00 2001
From: Mayur Singal <39544459+ulixius9@users.noreply.github.com>
Date: Wed, 29 Jun 2022 11:38:50 +0530
Subject: [PATCH] Added Include Tags in Ingestion UI (#5694)
* Added Include Tags in Ingestion UI
* Fix ConfigureIngestionProps
* Test Fix
* Fix Cypress
---
.../ingestion/source/database/bigquery.py | 4 ++++
.../AddIngestion/AddIngestion.component.tsx | 6 ++++++
.../Steps/ConfigureIngestion.test.tsx | 4 +++-
.../AddIngestion/Steps/ConfigureIngestion.tsx | 16 ++++++++++++++++
.../AddIngestion/addIngestion.interface.ts | 2 ++
.../ui/src/utils/PipelineServiceUtils.ts | 7 +++++++
6 files changed, 38 insertions(+), 1 deletion(-)
diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery.py b/ingestion/src/metadata/ingestion/source/database/bigquery.py
index b5ce961b38a..532628849ba 100644
--- a/ingestion/src/metadata/ingestion/source/database/bigquery.py
+++ b/ingestion/src/metadata/ingestion/source/database/bigquery.py
@@ -122,6 +122,8 @@ class BigquerySource(CommonDbSourceService):
:param _:
:return:
"""
+ if not self.source_config.includeTags:
+ return
taxonomies = PolicyTagManagerClient().list_taxonomies(
parent=f"projects/{self.project_id}/locations/{self.service_connection.taxonomyLocation}"
)
@@ -155,6 +157,8 @@ class BigquerySource(CommonDbSourceService):
This will only get executed if the tags context
is properly informed
"""
+ if not self.source_config.includeTags:
+ return
if column.get("policy_tags"):
return [
TagLabel(
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
index 7530196c43a..b5c45ed5101 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx
@@ -161,6 +161,9 @@ const AddIngestion = ({
const [includeView, setIncludeView] = useState(
Boolean((data?.sourceConfig.config as ConfigClass)?.includeViews)
);
+ const [includeTag, setIncludeTags] = useState(
+ Boolean((data?.sourceConfig.config as ConfigClass)?.includeTags)
+ );
const [includeLineage, setIncludeLineage] = useState(
Boolean((data?.sourceConfig.config as ConfigClass)?.includeLineage ?? true)
);
@@ -391,6 +394,7 @@ const AddIngestion = ({
return {
includeViews: includeView,
+ includeTags: includeTag,
databaseFilterPattern: getFilterPatternData(
databaseFilterPattern,
showDatabaseFilter
@@ -618,6 +622,7 @@ const AddIngestion = ({
handleDescription={(val) => setDescription(val)}
handleEnableDebugLog={() => setEnableDebugLog((pre) => !pre)}
handleIncludeLineage={() => setIncludeLineage((pre) => !pre)}
+ handleIncludeTags={() => setIncludeTags((pre) => !pre)}
handleIncludeView={() => setIncludeView((pre) => !pre)}
handleIngestSampleData={() => setIngestSampleData((pre) => !pre)}
handleIngestionName={(val) => setIngestionName(val)}
@@ -627,6 +632,7 @@ const AddIngestion = ({
handleShowFilter={handleShowFilter}
handleStageFileLocation={(val) => setStageFileLocation(val)}
includeLineage={includeLineage}
+ includeTags={includeTag}
includeView={includeView}
ingestSampleData={ingestSampleData}
ingestionName={ingestionName}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx
index ffab2650822..2b404978cfe 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx
@@ -64,6 +64,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
},
includeLineage: false,
includeView: false,
+ includeTags: false,
pipelineType: PipelineType.Metadata,
queryLogDuration: 1,
resultLimit: 100,
@@ -79,6 +80,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
showFqnFilter: false,
handleIncludeLineage: jest.fn(),
handleIncludeView: jest.fn(),
+ handleIncludeTags: jest.fn(),
handleIngestionName: jest.fn(),
handleMarkDeletedTables: jest.fn(),
handleQueryLogDuration: jest.fn(),
@@ -124,6 +126,6 @@ describe('Test ConfigureIngestion component', () => {
expect(backButton).toBeInTheDocument();
expect(nextButton).toBeInTheDocument();
expect(filterPatternComponents.length).toBe(3);
- expect(toggleSwitchs.length).toBe(3);
+ expect(toggleSwitchs.length).toBe(4);
});
});
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx
index 8a540fd6032..94b4f8b20b8 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx
@@ -39,6 +39,7 @@ const ConfigureIngestion = ({
fqnFilterPattern,
includeLineage,
includeView,
+ includeTags,
markDeletedTables,
serviceCategory,
pipelineType,
@@ -63,6 +64,7 @@ const ConfigureIngestion = ({
handleShowFilter,
handleIncludeLineage,
handleIncludeView,
+ handleIncludeTags,
handleMarkDeletedTables,
handleIngestSampleData,
handleDatasetServiceName,
@@ -128,6 +130,20 @@ const ConfigureIngestion = ({
{getSeparator('')}
+
+
+
+
+
+
+ Enable extracting tags from the data source
+
+ {getSeparator('')}
+
{getDebugLogToggle()}
{!isNil(markDeletedTables) && (
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts
index 8b0831e3739..97bc73e63ec 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts
@@ -69,6 +69,7 @@ export interface ConfigureIngestionProps {
fqnFilterPattern: FilterPattern;
includeLineage: boolean;
includeView: boolean;
+ includeTags: boolean;
markDeletedTables?: boolean;
enableDebugLog: boolean;
ingestSampleData: boolean;
@@ -89,6 +90,7 @@ export interface ConfigureIngestionProps {
handleDescription?: (value: string) => void;
handleIncludeLineage: () => void;
handleIncludeView: () => void;
+ handleIncludeTags: () => void;
handleMarkDeletedTables?: () => void;
handleEnableDebugLog: () => void;
handleIngestSampleData: () => void;
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/PipelineServiceUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/PipelineServiceUtils.ts
index d2e43e45a7d..bcab139a942 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/PipelineServiceUtils.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/PipelineServiceUtils.ts
@@ -14,6 +14,7 @@
import { cloneDeep } from 'lodash';
import { COMMON_UI_SCHEMA } from '../constants/services.const';
import { PipelineServiceType } from '../generated/entity/services/pipelineService';
+import airbyteConnection from '../jsons/connectionSchemas/connections/pipeline/airbyteConnection.json';
import airflowConnection from '../jsons/connectionSchemas/connections/pipeline/airflowConnection.json';
import glueConnection from '../jsons/connectionSchemas/connections/pipeline/glueConnection.json';
@@ -21,6 +22,12 @@ export const getPipelineConfig = (type: PipelineServiceType) => {
let schema = {};
const uiSchema = { ...COMMON_UI_SCHEMA };
switch (type) {
+ case PipelineServiceType.Airbyte: {
+ schema = airbyteConnection;
+
+ break;
+ }
+
case PipelineServiceType.Airflow: {
schema = airflowConnection;