UI : Dbt Workflow Improvement (#9404)

* Dbt Workflow Improvement

* added redirection to service type

* changes on localisation

openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx

@@ -25,6 +25,7 @@ import React, { Fragment, useEffect, useMemo, useState } from 'react';
 import { useHistory } from 'react-router-dom';
 import { PAGE_SIZE } from '../../constants/constants';
 import { WORKFLOWS_METADATA_DOCS } from '../../constants/docs.constants';
+import { PIPELINE_TYPE_LOCALISATION } from '../../constants/Ingestions.constant';
 import { MetadataServiceType } from '../../generated/api/services/createMetadataService';
 import { Connection } from '../../generated/entity/services/databaseService';
 import {
@@ -276,6 +277,32 @@ const Ingestion: React.FC<IngestionProps> = ({
     );
   };
 
+  const getAddIngestionName = (type: PipelineType): string => {
+    let name;
+    switch (type) {
+      case PipelineType.ElasticSearchReindex:
+        name = t('label.add-entity', {
+          entity: t('labe.elastic-search-re-index'),
+        });
+
+        break;
+
+      case PipelineType.Dbt:
+        name = t('label.add-workflow-ingestion', {
+          workflow: t('label.dbt-uppercase'),
+        });
+
+        break;
+
+      default:
+        name = t('label.add-workflow-ingestion', {
+          workflow: t(`label.${PIPELINE_TYPE_LOCALISATION[type]}`),
+        });
+    }
+
+    return name;
+  };
+
   const getAddIngestionDropdown = (types: PipelineType[]) => {
     return (
       <Fragment>
@@ -308,15 +335,11 @@ const Ingestion: React.FC<IngestionProps> = ({
           <DropDownList
             horzPosRight
             dropDownList={types.map((type) => ({
+              name: getAddIngestionName(type),
               disabled:
                 type === PipelineType.DataInsight
                   ? isDataSightIngestionExists
                   : false,
-              name: `${t('label.add')} ${startCase(type)} ${
-                type === PipelineType.ElasticSearchReindex
-                  ? ''
-                  : t('label.ingestion')
-              }`,
               value: type,
             }))}
             onSelect={(_e, value) =>

openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTFormConstants.ts

@@ -23,10 +23,6 @@ import {
 import { DBT_SOURCES, GCS_CONFIG } from './DBTFormEnum';
 
 export const DBTSources: Array<DropDownListItem> = [
-  {
-    name: 'No Config Source',
-    value: '',
-  },
   {
     name: 'Local Config Source',
     value: DBT_SOURCES.local,
@@ -66,12 +62,10 @@ export const reqDBTCloudFields: Record<keyof DbtConfigCloudReq, string> = {
 };
 
 export const reqDBTLocalFields: Record<string, string> = {
-  dbtCatalogFilePath: 'DBT Catalog File Path',
   dbtManifestFilePath: 'DBT Manifest File Path',
 };
 
 export const reqDBTHttpFields: Record<string, string> = {
-  dbtCatalogHttpPath: 'DBT Catalog Http Path',
   dbtManifestHttpPath: 'DBT Manifest Http Path',
 };
 
@@ -79,19 +73,6 @@ export const reqDBTS3Fields: Record<keyof DbtS3CredsReq, string> = {
   awsRegion: 'AWS Region',
 };
 
-export const reqDBTGCSCredsFields: Record<keyof DbtGCSCreds, string> = {
-  authProviderX509CertUrl: 'Authentication Provider x509 Certificate URL',
-  authUri: 'Authentication URI',
-  clientEmail: 'Client Email',
-  clientId: 'Client ID',
-  clientX509CertUrl: 'Client x509 Certificate URL',
-  privateKey: 'Private Key',
-  privateKeyId: 'Private Key ID',
-  projectId: 'Project ID',
-  tokenUri: 'Token URI',
-  type: 'Credentials Type',
-};
-
 export const rulesDBTS3CredsFields: Record<
   keyof Pick<FormValidationRules, FormValidationRulesType.url>,
   Array<keyof DbtS3Creds>

openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.test.tsx

@@ -416,15 +416,6 @@ describe('Test DBT GCS Config Form', () => {
     expect(mockPrefixConfigChange).toBeCalledTimes(2);
   });
 
-  it('should show errors on submit', async () => {
-    const { container } = render(<DBTGCSConfig {...mockProps} />);
-    const submitBtn = getByTestId(container, 'submit-btn');
-
-    fireEvent.click(submitBtn);
-
-    expect(mockSubmit).not.toBeCalled();
-  });
-
   it('should submit', async () => {
     const { container } = render(
       <DBTGCSConfig

openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.tsx

@@ -27,15 +27,7 @@ import {
   SCredentials,
 } from '../../../generated/metadataIngestion/dbtPipeline';
 import jsonData from '../../../jsons/en';
-import {
+import { errorMsg, getSeparator } from '../../../utils/CommonUtils';
-  errorMsg,
-  getSeparator,
-  requiredField,
-} from '../../../utils/CommonUtils';
-import {
-  checkDbtGCSCredsConfigRules,
-  validateDbtGCSCredsConfig,
-} from '../../../utils/DBTConfigFormUtil';
 import { Button } from '../../buttons/Button/Button';
 import { Field } from '../../Field/Field';
 import {
@@ -105,20 +97,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
   const validate = (data: DbtConfig) => {
     let valid = true;
     const gcsConfig = data.dbtSecurityConfig?.gcsConfig;
-    if (gcsType === GCS_CONFIG.GCSValues) {
+    if (gcsType !== GCS_CONFIG.GCSValues) {
-      const { isValid: reqValid, errors: reqErr } = validateDbtGCSCredsConfig(
-        (gcsConfig || {}) as GCSCredentialsValues
-      );
-      const { isValid: fieldValid, errors: fieldErr } =
-        checkDbtGCSCredsConfigRules((gcsConfig || {}) as GCSCredentialsValues);
-
-      setErrors({
-        ...fieldErr,
-        ...reqErr,
-      });
-
-      valid = reqValid && fieldValid;
-    } else {
       if (isEmpty(gcsConfig)) {
         setErrors({
           gcsConfig: `GCS Config ${jsonData['form-error-messages']['is-required']}`,
@@ -150,7 +129,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
           <label
             className="tw-block tw-form-label tw-mb-1"
             htmlFor="credential-type">
-            {requiredField('Credentials Type')}
+            Credentials Type
           </label>
           <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
             Google Cloud service account type.
@@ -170,7 +149,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
           <label
             className="tw-block tw-form-label tw-mb-1"
             htmlFor="project-id">
-            {requiredField('Project ID')}
+            Project ID
           </label>
           <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
             Google Cloud project id.
@@ -190,7 +169,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
           <label
             className="tw-block tw-form-label tw-mb-1"
             htmlFor="private-key-id">
-            {requiredField('Private Key ID')}
+            Private Key ID
           </label>
           <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
             Google Cloud Private key id.
@@ -211,7 +190,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
           <label
             className="tw-block tw-form-label tw-mb-1"
             htmlFor="private-key">
-            {requiredField('Private Key')}
+            Private Key
           </label>
           <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
             Google Cloud private key.
@@ -231,7 +210,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
           <label
             className="tw-block tw-form-label tw-mb-1"
             htmlFor="client-email">
-            {requiredField('Client Email')}
+            Client Email
           </label>
           <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
             Google Cloud email.
@@ -251,7 +230,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
         </Field>
         <Field>
           <label className="tw-block tw-form-label tw-mb-1" htmlFor="client-id">
-            {requiredField('Client ID')}
+            Client ID
           </label>
           <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
             Google Cloud Client ID.
@@ -269,7 +248,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
         </Field>
         <Field>
           <label className="tw-block tw-form-label tw-mb-1" htmlFor="auth-uri">
-            {requiredField('Authentication URI')}
+            Authentication URI
           </label>
           <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
             Google Cloud auth uri.
@@ -287,7 +266,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
         </Field>
         <Field>
           <label className="tw-block tw-form-label tw-mb-1" htmlFor="token-uri">
-            {requiredField('Token URI')}
+            Token URI
           </label>
           <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
             Google Cloud token uri.
@@ -307,7 +286,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
           <label
             className="tw-block tw-form-label tw-mb-1"
             htmlFor="auth-x509-certificate-uri">
-            {requiredField('Authentication Provider x509 Certificate URL')}
+            Authentication Provider x509 Certificate URL
           </label>
           <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
             Google Cloud auth provider certificate.
@@ -330,7 +309,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
           <label
             className="tw-block tw-form-label tw-mb-1"
             htmlFor="client-x509-certificate-uri">
-            {requiredField('Client x509 Certificate URL')}
+            Client x509 Certificate URL
           </label>
           <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
             Google Cloud client certificate uri.
@@ -358,7 +337,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
         <label
           className="tw-block tw-form-label tw-mb-1"
           htmlFor="gcs-cred-path">
-          {requiredField('GCS Credentials Path')}
+          GCS Credentials Path
         </label>
         <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
           GCS Credentials Path.

openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTHttpConfig.tsx

@@ -74,7 +74,7 @@ export const DBTHttpConfig: FunctionComponent<Props> = ({
     <Fragment>
       <Field>
         <label className="tw-block tw-form-label tw-mb-1" htmlFor="catalog-url">
-          {requiredField('DBT Catalog Http Path')}
+          DBT Catalog HTTP Path
         </label>
         <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
           DBT catalog file to extract dbt models with their column schemas.
@@ -94,7 +94,7 @@ export const DBTHttpConfig: FunctionComponent<Props> = ({
         <label
           className="tw-block tw-form-label tw-mb-1"
           htmlFor="manifest-url">
-          {requiredField('DBT Manifest Http Path')}
+          {requiredField('DBT Manifest HTTP Path')}
         </label>
         <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
           DBT manifest file path to extract dbt models and associate with
@@ -109,6 +109,7 @@ export const DBTHttpConfig: FunctionComponent<Props> = ({
           value={dbtManifestHttpPath}
           onChange={(e) => handleManifestHttpPathChange(e.target.value)}
         />
+        {errors?.dbtManifestHttpPath && errorMsg(errors.dbtManifestHttpPath)}
       </Field>
       <Field>
         <label

openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTLocalConfig.tsx

@@ -76,7 +76,7 @@ export const DBTLocalConfig: FunctionComponent<Props> = ({
         <label
           className="tw-block tw-form-label tw-mb-1"
           htmlFor="catalog-file">
-          {requiredField('DBT Catalog File Path')}
+          DBT Catalog File Path
         </label>
         <p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
           DBT catalog file to extract dbt models with their column schemas.

openmetadata-ui/src/main/resources/ui/src/constants/Ingestions.constant.ts

@@ -51,3 +51,19 @@ export const STEPS_FOR_ADD_SERVICE: Array<StepperStepType> = [
   },
   { name: i18next.t('label.connection-details'), step: 3 },
 ];
+
+export const INGESTION_ACTION_TYPE = {
+  ADD: 'add',
+  EDIT: 'edit',
+};
+
+export const PIPELINE_TYPE_LOCALISATION = {
+  dataInsight: 'data-insight',
+  dbt: 'dbt',
+  elasticSearchReindex: 'elastic-search-re-index',
+  lineage: 'lineage',
+  metadata: 'metadata',
+  profiler: 'profiler',
+  TestSuite: 'test-suite',
+  usage: 'usage',
+};

openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts

@@ -190,6 +190,15 @@ export const SERVICE_CATEGORY: { [key: string]: ServiceCategory } = {
   metadata: ServiceCategory.METADATA_SERVICES,
 };
 
+export const SERVICE_CATEGORY_TYPE = {
+  databaseServices: 'databases',
+  messagingServices: 'messaging',
+  dashboardServices: 'dashboards',
+  pipelineServices: 'pipelines',
+  mlmodelServices: 'mlModels',
+  metadataServices: 'metadata',
+};
+
 export const servicesDisplayName: { [key: string]: string } = {
   databaseServices: i18next.t('label.entity-service', {
     entity: i18next.t('label.database'),

openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json

@@ -309,6 +309,7 @@
     "detail-plural": "Details",
     "ingestion": "Ingestion",
     "add-workflow-ingestion": "Add {{workflow}} Ingestion",
+    "edit-workflow-ingestion": "Edit {{workflow}} Ingestion",
     "clear-all": "Clear All",
     "view-less": "View less",
     "view-more": "View more",
@@ -584,6 +585,7 @@
     "assigned-entity": "Assigned {{entity}}",
     "total-assets-view": "Total Assets View",
     "total-active-user": "Total Active User",
+    "elastic-search-re-index": "ElasticSearchReindex",
     "alert-actions": "Alert Actions",
     "test-results": "Test Results",
     "send-to": "Send to",

openmetadata-ui/src/main/resources/ui/src/pages/AddIngestionPage/AddIngestionPage.component.tsx

@@ -13,7 +13,7 @@
 
 import { Space } from 'antd';
 import { AxiosError } from 'axios';
-import { capitalize, startCase } from 'lodash';
+import { startCase } from 'lodash';
 import { ServiceTypes } from 'Models';
 import React, { useEffect, useState } from 'react';
 import { useTranslation } from 'react-i18next';
@@ -39,6 +39,7 @@ import {
   INGESTION_PROGRESS_START_VAL,
 } from '../../constants/constants';
 import { GlobalSettingsMenuCategory } from '../../constants/GlobalSettings.constants';
+import { INGESTION_ACTION_TYPE } from '../../constants/Ingestions.constant';
 import { FormSubmitType } from '../../enums/form.enum';
 import { IngestionActionMessage } from '../../enums/ingestion.enum';
 import { ServiceCategory } from '../../enums/service.enum';
@@ -47,6 +48,7 @@ import { PipelineType } from '../../generated/entity/services/ingestionPipelines
 import { DataObj } from '../../interface/service.interface';
 import jsonData from '../../jsons/en';
 import { getEntityMissingError } from '../../utils/CommonUtils';
+import { getIngestionHeadingName } from '../../utils/IngestionUtils';
 import { getSettingPath } from '../../utils/RouterUtils';
 import {
   getServiceIngestionStepGuide,
@@ -266,7 +268,10 @@ const AddIngestionPage = () => {
                   activeIngestionStep={activeIngestionStep}
                   handleCancelClick={goToService}
                   handleViewServiceClick={goToService}
-                  heading={`Add ${capitalize(ingestionType)} Ingestion`}
+                  heading={getIngestionHeadingName(
+                    ingestionType,
+                    INGESTION_ACTION_TYPE.ADD
+                  )}
                   ingestionAction={ingestionAction}
                   ingestionProgress={ingestionProgress}
                   isAirflowSetup={isAirflowRunning}

openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx

@@ -38,6 +38,7 @@ import {
   INGESTION_PROGRESS_START_VAL,
 } from '../../constants/constants';
 import { GlobalSettingsMenuCategory } from '../../constants/GlobalSettings.constants';
+import { INGESTION_ACTION_TYPE } from '../../constants/Ingestions.constant';
 import { FormSubmitType } from '../../enums/form.enum';
 import { IngestionActionMessage } from '../../enums/ingestion.enum';
 import { ServiceCategory } from '../../enums/service.enum';
@@ -49,6 +50,7 @@ import {
 import { DataObj } from '../../interface/service.interface';
 import jsonData from '../../jsons/en';
 import { getEntityMissingError } from '../../utils/CommonUtils';
+import { getIngestionHeadingName } from '../../utils/IngestionUtils';
 import { getSettingPath } from '../../utils/RouterUtils';
 import {
   getServiceIngestionStepGuide,
@@ -297,7 +299,10 @@ const EditIngestionPage = () => {
                   data={ingestionData}
                   handleCancelClick={goToService}
                   handleViewServiceClick={goToService}
-                  heading={`Edit ${capitalize(ingestionType)} Ingestion`}
+                  heading={getIngestionHeadingName(
+                    ingestionType,
+                    INGESTION_ACTION_TYPE.EDIT
+                  )}
                   ingestionAction={ingestionAction}
                   ingestionProgress={ingestionProgress}
                   isAirflowSetup={isAirflowRunning}

openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx

@@ -68,6 +68,7 @@ import { GlobalSettingsMenuCategory } from '../../constants/GlobalSettings.const
 import {
   OPENMETADATA,
   servicesDisplayName,
+  SERVICE_CATEGORY_TYPE,
 } from '../../constants/Services.constant';
 import { SearchIndex } from '../../enums/search.enum';
 import { ServiceCategory } from '../../enums/service.enum';
@@ -1014,6 +1015,15 @@ const ServicePage: FunctionComponent = () => {
                   )}
                   <DeleteWidgetModal
                     isRecursiveDelete
+                    afterDeleteAction={() =>
+                      history.push(
+                        `/settings/services/${
+                          SERVICE_CATEGORY_TYPE[
+                            serviceCategory as keyof typeof SERVICE_CATEGORY_TYPE
+                          ]
+                        }`
+                      )
+                    }
                     allowSoftDelete={false}
                     deleteMessage={getDeleteEntityMessage(
                       serviceName || '',

openmetadata-ui/src/main/resources/ui/src/utils/DBTConfigFormUtil.ts

@@ -17,7 +17,6 @@ import {
   DbtConfigCloudReq,
   DbtConfigHttp,
   DbtConfigLocal,
-  DbtGCSCreds,
   DbtS3CredsReq,
   DbtSourceTypes,
   ErrorDbtCloud,
@@ -28,7 +27,6 @@ import {
 } from '../components/common/DBTConfigFormBuilder/DBTConfigForm.interface';
 import {
   reqDBTCloudFields,
-  reqDBTGCSCredsFields,
   reqDBTHttpFields,
   reqDBTLocalFields,
   reqDBTS3Fields,
@@ -129,24 +127,6 @@ export const validateDbtS3Config = (
   return { isValid, errors };
 };
 
-export const validateDbtGCSCredsConfig = (
-  data: GCSCredentialsValues,
-  requiredFields = reqDBTGCSCredsFields
-) => {
-  let isValid = true;
-  const errors = {} as ErrorDbtGCS;
-  for (const field of Object.keys(requiredFields) as Array<keyof DbtGCSCreds>) {
-    if (isEmpty(data[field])) {
-      isValid = false;
-      errors[
-        field
-      ] = `${requiredFields[field]} ${jsonData['form-error-messages']['is-required']}`;
-    }
-  }
-
-  return { isValid, errors };
-};
-
 function getInvalidEmailErrors<
   Type,
   Keys extends Array<keyof Type>,
@@ -250,7 +230,7 @@ export const checkDbtGCSCredsConfigRules = (
 
 export const getSourceTypeFromConfig = (
   data?: DbtConfig,
-  defaultSource = '' as DBT_SOURCES
+  defaultSource = DBT_SOURCES.local
 ): DbtSourceTypes => {
   let sourceType = defaultSource;
   let gcsType = undefined;

openmetadata-ui/src/main/resources/ui/src/utils/IngestionUtils.ts

@@ -0,0 +1,36 @@
+/*
+ *  Copyright 2022 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+import { t } from 'i18next';
+
+import { capitalize, upperCase } from 'lodash';
+import { INGESTION_ACTION_TYPE } from '../constants/Ingestions.constant';
+import { PipelineType } from '../generated/api/services/ingestionPipelines/createIngestionPipeline';
+
+export const getIngestionHeadingName = (
+  ingestionType: string,
+  type: string
+) => {
+  let ingestionName = capitalize(ingestionType);
+  if (ingestionType === PipelineType.Dbt) {
+    ingestionName = upperCase(ingestionType);
+  }
+
+  return type === INGESTION_ACTION_TYPE.ADD
+    ? t('label.add-workflow-ingestion', {
+        workflow: ingestionName,
+      })
+    : t('label.edit-workflow-ingestion', {
+        workflow: ingestionName,
+      });
+};