Added Include Tags in Ingestion UI (#5694)

* Added Include Tags in Ingestion UI

* Fix ConfigureIngestionProps

* Test Fix

* Fix Cypress

ingestion/src/metadata/ingestion/source/database/bigquery.py

@@ -122,6 +122,8 @@ class BigquerySource(CommonDbSourceService):
         :param _:
         :return:
         """
+        if not self.source_config.includeTags:
+            return
         taxonomies = PolicyTagManagerClient().list_taxonomies(
             parent=f"projects/{self.project_id}/locations/{self.service_connection.taxonomyLocation}"
         )
@@ -155,6 +157,8 @@ class BigquerySource(CommonDbSourceService):
         This will only get executed if the tags context
         is properly informed
         """
+        if not self.source_config.includeTags:
+            return
         if column.get("policy_tags"):
             return [
                 TagLabel(

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/AddIngestion.component.tsx

@@ -161,6 +161,9 @@ const AddIngestion = ({
   const [includeView, setIncludeView] = useState(
     Boolean((data?.sourceConfig.config as ConfigClass)?.includeViews)
   );
+  const [includeTag, setIncludeTags] = useState(
+    Boolean((data?.sourceConfig.config as ConfigClass)?.includeTags)
+  );
   const [includeLineage, setIncludeLineage] = useState(
     Boolean((data?.sourceConfig.config as ConfigClass)?.includeLineage ?? true)
   );
@@ -391,6 +394,7 @@ const AddIngestion = ({
 
         return {
           includeViews: includeView,
+          includeTags: includeTag,
           databaseFilterPattern: getFilterPatternData(
             databaseFilterPattern,
             showDatabaseFilter
@@ -618,6 +622,7 @@ const AddIngestion = ({
             handleDescription={(val) => setDescription(val)}
             handleEnableDebugLog={() => setEnableDebugLog((pre) => !pre)}
             handleIncludeLineage={() => setIncludeLineage((pre) => !pre)}
+            handleIncludeTags={() => setIncludeTags((pre) => !pre)}
             handleIncludeView={() => setIncludeView((pre) => !pre)}
             handleIngestSampleData={() => setIngestSampleData((pre) => !pre)}
             handleIngestionName={(val) => setIngestionName(val)}
@@ -627,6 +632,7 @@ const AddIngestion = ({
             handleShowFilter={handleShowFilter}
             handleStageFileLocation={(val) => setStageFileLocation(val)}
             includeLineage={includeLineage}
+            includeTags={includeTag}
             includeView={includeView}
             ingestSampleData={ingestSampleData}
             ingestionName={ingestionName}

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.test.tsx

@@ -64,6 +64,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
   },
   includeLineage: false,
   includeView: false,
+  includeTags: false,
   pipelineType: PipelineType.Metadata,
   queryLogDuration: 1,
   resultLimit: 100,
@@ -79,6 +80,7 @@ const mockConfigureIngestion: ConfigureIngestionProps = {
   showFqnFilter: false,
   handleIncludeLineage: jest.fn(),
   handleIncludeView: jest.fn(),
+  handleIncludeTags: jest.fn(),
   handleIngestionName: jest.fn(),
   handleMarkDeletedTables: jest.fn(),
   handleQueryLogDuration: jest.fn(),
@@ -124,6 +126,6 @@ describe('Test ConfigureIngestion component', () => {
     expect(backButton).toBeInTheDocument();
     expect(nextButton).toBeInTheDocument();
     expect(filterPatternComponents.length).toBe(3);
-    expect(toggleSwitchs.length).toBe(3);
+    expect(toggleSwitchs.length).toBe(4);
   });
 });

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/Steps/ConfigureIngestion.tsx

@@ -39,6 +39,7 @@ const ConfigureIngestion = ({
   fqnFilterPattern,
   includeLineage,
   includeView,
+  includeTags,
   markDeletedTables,
   serviceCategory,
   pipelineType,
@@ -63,6 +64,7 @@ const ConfigureIngestion = ({
   handleShowFilter,
   handleIncludeLineage,
   handleIncludeView,
+  handleIncludeTags,
   handleMarkDeletedTables,
   handleIngestSampleData,
   handleDatasetServiceName,
@@ -128,6 +130,20 @@ const ConfigureIngestion = ({
             </p>
             {getSeparator('')}
           </Field>
+          <Field>
+            <div className="tw-flex tw-gap-1">
+              <label>Include tags</label>
+              <ToggleSwitchV1
+                checked={includeTags}
+                handleCheck={handleIncludeTags}
+                testId="include-tags"
+              />
+            </div>
+            <p className="tw-text-grey-muted tw-mt-3">
+              Enable extracting tags from the data source
+            </p>
+            {getSeparator('')}
+          </Field>
           {getDebugLogToggle()}
           {!isNil(markDeletedTables) && (
             <Field>

openmetadata-ui/src/main/resources/ui/src/components/AddIngestion/addIngestion.interface.ts

@@ -69,6 +69,7 @@ export interface ConfigureIngestionProps {
   fqnFilterPattern: FilterPattern;
   includeLineage: boolean;
   includeView: boolean;
+  includeTags: boolean;
   markDeletedTables?: boolean;
   enableDebugLog: boolean;
   ingestSampleData: boolean;
@@ -89,6 +90,7 @@ export interface ConfigureIngestionProps {
   handleDescription?: (value: string) => void;
   handleIncludeLineage: () => void;
   handleIncludeView: () => void;
+  handleIncludeTags: () => void;
   handleMarkDeletedTables?: () => void;
   handleEnableDebugLog: () => void;
   handleIngestSampleData: () => void;

openmetadata-ui/src/main/resources/ui/src/utils/PipelineServiceUtils.ts

@@ -14,6 +14,7 @@
 import { cloneDeep } from 'lodash';
 import { COMMON_UI_SCHEMA } from '../constants/services.const';
 import { PipelineServiceType } from '../generated/entity/services/pipelineService';
+import airbyteConnection from '../jsons/connectionSchemas/connections/pipeline/airbyteConnection.json';
 import airflowConnection from '../jsons/connectionSchemas/connections/pipeline/airflowConnection.json';
 import glueConnection from '../jsons/connectionSchemas/connections/pipeline/glueConnection.json';
 
@@ -21,6 +22,12 @@ export const getPipelineConfig = (type: PipelineServiceType) => {
   let schema = {};
   const uiSchema = { ...COMMON_UI_SCHEMA };
   switch (type) {
+    case PipelineServiceType.Airbyte: {
+      schema = airbyteConnection;
+
+      break;
+    }
+
     case PipelineServiceType.Airflow: {
       schema = airflowConnection;