yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-01 11:09:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

UI : Dbt Workflow Improvement (#9404)

Browse Source 
* Dbt Workflow Improvement

* added redirection to service type

* changes on localisation
This commit is contained in:
Ashish Gupta 2022-12-20 11:48:50 +05:30 committed by GitHub
parent ba0f55d7c3
commit 6eccab6157
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 132 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
openmetadata-ui/src/main/resources/ui/src/components/Ingestion/Ingestion.component.tsx
View File

@ -25,6 +25,7 @@ import React, { Fragment, useEffect, useMemo, useState } from 'react';
import { useHistory } from 'react-router-dom';
import { PAGE_SIZE } from '../../constants/constants';
import { WORKFLOWS_METADATA_DOCS } from '../../constants/docs.constants';
import { PIPELINE_TYPE_LOCALISATION } from '../../constants/Ingestions.constant';
import { MetadataServiceType } from '../../generated/api/services/createMetadataService';
import { Connection } from '../../generated/entity/services/databaseService';
import {
@ -276,6 +277,32 @@ const Ingestion: React.FC<IngestionProps> = ({
);
};
const getAddIngestionName = (type: PipelineType): string => {
let name;
switch (type) {
case PipelineType.ElasticSearchReindex:
name = t('label.add-entity', {
entity: t('labe.elastic-search-re-index'),
});
break;
case PipelineType.Dbt:
name = t('label.add-workflow-ingestion', {
workflow: t('label.dbt-uppercase'),
});
break;
default:
name = t('label.add-workflow-ingestion', {
workflow: t(`label.${PIPELINE_TYPE_LOCALISATION[type]}`),
});
}
return name;
};
const getAddIngestionDropdown = (types: PipelineType[]) => {
return (
<Fragment>
@ -308,15 +335,11 @@ const Ingestion: React.FC<IngestionProps> = ({
<DropDownList
horzPosRight
dropDownList={types.map((type) => ({
name: getAddIngestionName(type),
disabled:
type === PipelineType.DataInsight
? isDataSightIngestionExists
: false,
name: `${t('label.add')} ${startCase(type)} ${
type === PipelineType.ElasticSearchReindex
? ''
: t('label.ingestion')
}`,
value: type,
}))}
onSelect={(_e, value) =>

19
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTFormConstants.ts
View File

@ -23,10 +23,6 @@ import {
import { DBT_SOURCES, GCS_CONFIG } from './DBTFormEnum';
export const DBTSources: Array<DropDownListItem> = [
{
name: 'No Config Source',
value: '',
},
{
name: 'Local Config Source',
value: DBT_SOURCES.local,
@ -66,12 +62,10 @@ export const reqDBTCloudFields: Record<keyof DbtConfigCloudReq, string> = {
};
export const reqDBTLocalFields: Record<string, string> = {
dbtCatalogFilePath: 'DBT Catalog File Path',
dbtManifestFilePath: 'DBT Manifest File Path',
};
export const reqDBTHttpFields: Record<string, string> = {
dbtCatalogHttpPath: 'DBT Catalog Http Path',
dbtManifestHttpPath: 'DBT Manifest Http Path',
};
@ -79,19 +73,6 @@ export const reqDBTS3Fields: Record<keyof DbtS3CredsReq, string> = {
awsRegion: 'AWS Region',
};
export const reqDBTGCSCredsFields: Record<keyof DbtGCSCreds, string> = {
authProviderX509CertUrl: 'Authentication Provider x509 Certificate URL',
authUri: 'Authentication URI',
clientEmail: 'Client Email',
clientId: 'Client ID',
clientX509CertUrl: 'Client x509 Certificate URL',
privateKey: 'Private Key',
privateKeyId: 'Private Key ID',
projectId: 'Project ID',
tokenUri: 'Token URI',
type: 'Credentials Type',
};
export const rulesDBTS3CredsFields: Record<
keyof Pick<FormValidationRules, FormValidationRulesType.url>,
Array<keyof DbtS3Creds>

9
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.test.tsx
View File

@ -416,15 +416,6 @@ describe('Test DBT GCS Config Form', () => {
expect(mockPrefixConfigChange).toBeCalledTimes(2);
});
it('should show errors on submit', async () => {
const { container } = render(<DBTGCSConfig {...mockProps} />);
const submitBtn = getByTestId(container, 'submit-btn');
fireEvent.click(submitBtn);
expect(mockSubmit).not.toBeCalled();
});
it('should submit', async () => {
const { container } = render(
<DBTGCSConfig

47
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTGCSConfig.tsx
View File

@ -27,15 +27,7 @@ import {
SCredentials,
} from '../../../generated/metadataIngestion/dbtPipeline';
import jsonData from '../../../jsons/en';
import {
errorMsg,
getSeparator,
requiredField,
} from '../../../utils/CommonUtils';
import {
checkDbtGCSCredsConfigRules,
validateDbtGCSCredsConfig,
} from '../../../utils/DBTConfigFormUtil';
import { errorMsg, getSeparator } from '../../../utils/CommonUtils';
import { Button } from '../../buttons/Button/Button';
import { Field } from '../../Field/Field';
import {
@ -105,20 +97,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
const validate = (data: DbtConfig) => {
let valid = true;
const gcsConfig = data.dbtSecurityConfig?.gcsConfig;
if (gcsType === GCS_CONFIG.GCSValues) {
const { isValid: reqValid, errors: reqErr } = validateDbtGCSCredsConfig(
(gcsConfig || {}) as GCSCredentialsValues
);
const { isValid: fieldValid, errors: fieldErr } =
checkDbtGCSCredsConfigRules((gcsConfig || {}) as GCSCredentialsValues);
setErrors({
...fieldErr,
...reqErr,
});
valid = reqValid && fieldValid;
} else {
if (gcsType !== GCS_CONFIG.GCSValues) {
if (isEmpty(gcsConfig)) {
setErrors({
gcsConfig: `GCS Config ${jsonData['form-error-messages']['is-required']}`,
@ -150,7 +129,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="credential-type">
{requiredField('Credentials Type')}
Credentials Type
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Google Cloud service account type.
@ -170,7 +149,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="project-id">
{requiredField('Project ID')}
Project ID
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Google Cloud project id.
@ -190,7 +169,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="private-key-id">
{requiredField('Private Key ID')}
Private Key ID
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Google Cloud Private key id.
@ -211,7 +190,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="private-key">
{requiredField('Private Key')}
Private Key
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Google Cloud private key.
@ -231,7 +210,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="client-email">
{requiredField('Client Email')}
Client Email
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Google Cloud email.
@ -251,7 +230,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
</Field>
<Field>
<label className="tw-block tw-form-label tw-mb-1" htmlFor="client-id">
{requiredField('Client ID')}
Client ID
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Google Cloud Client ID.
@ -269,7 +248,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
</Field>
<Field>
<label className="tw-block tw-form-label tw-mb-1" htmlFor="auth-uri">
{requiredField('Authentication URI')}
Authentication URI
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Google Cloud auth uri.
@ -287,7 +266,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
</Field>
<Field>
<label className="tw-block tw-form-label tw-mb-1" htmlFor="token-uri">
{requiredField('Token URI')}
Token URI
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Google Cloud token uri.
@ -307,7 +286,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="auth-x509-certificate-uri">
{requiredField('Authentication Provider x509 Certificate URL')}
Authentication Provider x509 Certificate URL
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Google Cloud auth provider certificate.
@ -330,7 +309,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="client-x509-certificate-uri">
{requiredField('Client x509 Certificate URL')}
Client x509 Certificate URL
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
Google Cloud client certificate uri.
@ -358,7 +337,7 @@ export const DBTGCSConfig: FunctionComponent<Props> = ({
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="gcs-cred-path">
{requiredField('GCS Credentials Path')}
GCS Credentials Path
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
GCS Credentials Path.

5
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTHttpConfig.tsx
View File

@ -74,7 +74,7 @@ export const DBTHttpConfig: FunctionComponent<Props> = ({
<Fragment>
<Field>
<label className="tw-block tw-form-label tw-mb-1" htmlFor="catalog-url">
{requiredField('DBT Catalog Http Path')}
DBT Catalog HTTP Path
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
DBT catalog file to extract dbt models with their column schemas.
@ -94,7 +94,7 @@ export const DBTHttpConfig: FunctionComponent<Props> = ({
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="manifest-url">
{requiredField('DBT Manifest Http Path')}
{requiredField('DBT Manifest HTTP Path')}
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
DBT manifest file path to extract dbt models and associate with
@ -109,6 +109,7 @@ export const DBTHttpConfig: FunctionComponent<Props> = ({
value={dbtManifestHttpPath}
onChange={(e) => handleManifestHttpPathChange(e.target.value)}
/>
{errors?.dbtManifestHttpPath && errorMsg(errors.dbtManifestHttpPath)}
</Field>
<Field>
<label

2
openmetadata-ui/src/main/resources/ui/src/components/common/DBTConfigFormBuilder/DBTLocalConfig.tsx
View File

@ -76,7 +76,7 @@ export const DBTLocalConfig: FunctionComponent<Props> = ({
<label
className="tw-block tw-form-label tw-mb-1"
htmlFor="catalog-file">
{requiredField('DBT Catalog File Path')}
DBT Catalog File Path
</label>
<p className="tw-text-grey-muted tw-mt-1 tw-mb-2 tw-text-xs">
DBT catalog file to extract dbt models with their column schemas.

16
openmetadata-ui/src/main/resources/ui/src/constants/Ingestions.constant.ts
View File

@ -51,3 +51,19 @@ export const STEPS_FOR_ADD_SERVICE: Array<StepperStepType> = [
},
{ name: i18next.t('label.connection-details'), step: 3 },
];
export const INGESTION_ACTION_TYPE = {
ADD: 'add',
EDIT: 'edit',
};
export const PIPELINE_TYPE_LOCALISATION = {
dataInsight: 'data-insight',
dbt: 'dbt',
elasticSearchReindex: 'elastic-search-re-index',
lineage: 'lineage',
metadata: 'metadata',
profiler: 'profiler',
TestSuite: 'test-suite',
usage: 'usage',
};

9
openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts
View File

@ -190,6 +190,15 @@ export const SERVICE_CATEGORY: { [key: string]: ServiceCategory } = {
metadata: ServiceCategory.METADATA_SERVICES,
};
export const SERVICE_CATEGORY_TYPE = {
databaseServices: 'databases',
messagingServices: 'messaging',
dashboardServices: 'dashboards',
pipelineServices: 'pipelines',
mlmodelServices: 'mlModels',
metadataServices: 'metadata',
};
export const servicesDisplayName: { [key: string]: string } = {
databaseServices: i18next.t('label.entity-service', {
entity: i18next.t('label.database'),

2
openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json
View File

@ -309,6 +309,7 @@
"detail-plural": "Details",
"ingestion": "Ingestion",
"add-workflow-ingestion": "Add {{workflow}} Ingestion",
"edit-workflow-ingestion": "Edit {{workflow}} Ingestion",
"clear-all": "Clear All",
"view-less": "View less",
"view-more": "View more",
@ -584,6 +585,7 @@
"assigned-entity": "Assigned {{entity}}",
"total-assets-view": "Total Assets View",
"total-active-user": "Total Active User",
"elastic-search-re-index": "ElasticSearchReindex",
"alert-actions": "Alert Actions",
"test-results": "Test Results",
"send-to": "Send to",

9
openmetadata-ui/src/main/resources/ui/src/pages/AddIngestionPage/AddIngestionPage.component.tsx
View File

@ -13,7 +13,7 @@
import { Space } from 'antd';
import { AxiosError } from 'axios';
import { capitalize, startCase } from 'lodash';
import { startCase } from 'lodash';
import { ServiceTypes } from 'Models';
import React, { useEffect, useState } from 'react';
import { useTranslation } from 'react-i18next';
@ -39,6 +39,7 @@ import {
INGESTION_PROGRESS_START_VAL,
} from '../../constants/constants';
import { GlobalSettingsMenuCategory } from '../../constants/GlobalSettings.constants';
import { INGESTION_ACTION_TYPE } from '../../constants/Ingestions.constant';
import { FormSubmitType } from '../../enums/form.enum';
import { IngestionActionMessage } from '../../enums/ingestion.enum';
import { ServiceCategory } from '../../enums/service.enum';
@ -47,6 +48,7 @@ import { PipelineType } from '../../generated/entity/services/ingestionPipelines
import { DataObj } from '../../interface/service.interface';
import jsonData from '../../jsons/en';
import { getEntityMissingError } from '../../utils/CommonUtils';
import { getIngestionHeadingName } from '../../utils/IngestionUtils';
import { getSettingPath } from '../../utils/RouterUtils';
import {
getServiceIngestionStepGuide,
@ -266,7 +268,10 @@ const AddIngestionPage = () => {
activeIngestionStep={activeIngestionStep}
handleCancelClick={goToService}
handleViewServiceClick={goToService}
heading={`Add ${capitalize(ingestionType)} Ingestion`}
heading={getIngestionHeadingName(
ingestionType,
INGESTION_ACTION_TYPE.ADD
)}
ingestionAction={ingestionAction}
ingestionProgress={ingestionProgress}
isAirflowSetup={isAirflowRunning}

7
openmetadata-ui/src/main/resources/ui/src/pages/EditIngestionPage/EditIngestionPage.component.tsx
View File

@ -38,6 +38,7 @@ import {
INGESTION_PROGRESS_START_VAL,
} from '../../constants/constants';
import { GlobalSettingsMenuCategory } from '../../constants/GlobalSettings.constants';
import { INGESTION_ACTION_TYPE } from '../../constants/Ingestions.constant';
import { FormSubmitType } from '../../enums/form.enum';
import { IngestionActionMessage } from '../../enums/ingestion.enum';
import { ServiceCategory } from '../../enums/service.enum';
@ -49,6 +50,7 @@ import {
import { DataObj } from '../../interface/service.interface';
import jsonData from '../../jsons/en';
import { getEntityMissingError } from '../../utils/CommonUtils';
import { getIngestionHeadingName } from '../../utils/IngestionUtils';
import { getSettingPath } from '../../utils/RouterUtils';
import {
getServiceIngestionStepGuide,
@ -297,7 +299,10 @@ const EditIngestionPage = () => {
data={ingestionData}
handleCancelClick={goToService}
handleViewServiceClick={goToService}
heading={`Edit ${capitalize(ingestionType)} Ingestion`}
heading={getIngestionHeadingName(
ingestionType,
INGESTION_ACTION_TYPE.EDIT
)}
ingestionAction={ingestionAction}
ingestionProgress={ingestionProgress}
isAirflowSetup={isAirflowRunning}

10
openmetadata-ui/src/main/resources/ui/src/pages/service/index.tsx
View File

@ -68,6 +68,7 @@ import { GlobalSettingsMenuCategory } from '../../constants/GlobalSettings.const
import {
OPENMETADATA,
servicesDisplayName,
SERVICE_CATEGORY_TYPE,
} from '../../constants/Services.constant';
import { SearchIndex } from '../../enums/search.enum';
import { ServiceCategory } from '../../enums/service.enum';
@ -1014,6 +1015,15 @@ const ServicePage: FunctionComponent = () => {
)}
<DeleteWidgetModal
isRecursiveDelete
afterDeleteAction={() =>
history.push(
`/settings/services/${
SERVICE_CATEGORY_TYPE[
serviceCategory as keyof typeof SERVICE_CATEGORY_TYPE
]
}`
)
}
allowSoftDelete={false}
deleteMessage={getDeleteEntityMessage(
serviceName || '',

22
openmetadata-ui/src/main/resources/ui/src/utils/DBTConfigFormUtil.ts
View File

@ -17,7 +17,6 @@ import {
DbtConfigCloudReq,
DbtConfigHttp,
DbtConfigLocal,
DbtGCSCreds,
DbtS3CredsReq,
DbtSourceTypes,
ErrorDbtCloud,
@ -28,7 +27,6 @@ import {
} from '../components/common/DBTConfigFormBuilder/DBTConfigForm.interface';
import {
reqDBTCloudFields,
reqDBTGCSCredsFields,
reqDBTHttpFields,
reqDBTLocalFields,
reqDBTS3Fields,
@ -129,24 +127,6 @@ export const validateDbtS3Config = (
return { isValid, errors };
};
export const validateDbtGCSCredsConfig = (
data: GCSCredentialsValues,
requiredFields = reqDBTGCSCredsFields
) => {
let isValid = true;
const errors = {} as ErrorDbtGCS;
for (const field of Object.keys(requiredFields) as Array<keyof DbtGCSCreds>) {
if (isEmpty(data[field])) {
isValid = false;
errors[
field
] = `${requiredFields[field]} ${jsonData['form-error-messages']['is-required']}`;
}
}
return { isValid, errors };
};
function getInvalidEmailErrors<
Type,
Keys extends Array<keyof Type>,
@ -250,7 +230,7 @@ export const checkDbtGCSCredsConfigRules = (
export const getSourceTypeFromConfig = (
data?: DbtConfig,
defaultSource = '' as DBT_SOURCES
defaultSource = DBT_SOURCES.local
): DbtSourceTypes => {
let sourceType = defaultSource;
let gcsType = undefined;

36
openmetadata-ui/src/main/resources/ui/src/utils/IngestionUtils.ts Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { t } from 'i18next';
import { capitalize, upperCase } from 'lodash';
import { INGESTION_ACTION_TYPE } from '../constants/Ingestions.constant';
import { PipelineType } from '../generated/api/services/ingestionPipelines/createIngestionPipeline';
export const getIngestionHeadingName = (
ingestionType: string,
type: string
) => {
let ingestionName = capitalize(ingestionType);
if (ingestionType === PipelineType.Dbt) {
ingestionName = upperCase(ingestionType);
}
return type === INGESTION_ACTION_TYPE.ADD
? t('label.add-workflow-ingestion', {
workflow: ingestionName,
})
: t('label.edit-workflow-ingestion', {
workflow: ingestionName,
});
};