yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-03 03:59:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(ui): glossary import optimization around files under BulkImportPage (#20991)

Browse Source 
* glossary import optimization around files under BulkImportPage

* fix unit test

* fix the special character glossary breaking for import
This commit is contained in:
Ashish Gupta 2025-04-29 09:50:31 +05:30 committed by GitHub
parent 1447767d0c
commit b3d7f97590
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 119 additions and 975 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts
View File

@ -155,7 +155,7 @@ test.describe('Glossary Bulk Import Export', () => {
await page.click('[data-testid="manage-button"]');
await page.click('[data-testid="import-button-description"]');
const fileInput = await page.$('[type="file"]');
const fileInput = page.getByTestId('upload-file-widget');
await fileInput?.setInputFiles([
'downloads/' + glossary1.data.displayName + '.csv',
]);

2
openmetadata-ui/src/main/resources/ui/src/components/AppRouter/EntityImportRouter.tsx
View File

@ -35,8 +35,6 @@ const EntityImportRouter = () => {
setIsLoading(true);
const entityPermission = await getEntityPermissionByFqn(entityType, fqn);
setEntityPermission(entityPermission);
} catch (error) {
// will not show logs
} finally {
setIsLoading(false);
}

3
openmetadata-ui/src/main/resources/ui/src/components/BulkEditEntity/BulkEditEntity.component.tsx
View File

@ -23,7 +23,6 @@ import { EntityType } from '../../enums/entity.enum';
import { useFqn } from '../../hooks/useFqn';
import { getBulkEditCSVExportEntityApi } from '../../utils/EntityBulkEdit/EntityBulkEditUtils';
import entityUtilClassBase from '../../utils/EntityUtilClassBase';
import { getEncodedFqn } from '../../utils/StringsUtils';
import Banner from '../common/Banner/Banner';
import { ImportStatus } from '../common/EntityImport/ImportStatus/ImportStatus.component';
import Loader from '../common/Loader/Loader';
@ -64,7 +63,7 @@ const BulkEditEntity = ({
useEffect(() => {
triggerExportForBulkEdit({
name: getEncodedFqn(fqn),
name: fqn,
onExport: getBulkEditCSVExportEntityApi(entityType),
exportTypes: [ExportTypes.CSV],
});

2
openmetadata-ui/src/main/resources/ui/src/components/BulkEditEntity/BulkEditEntity.interface.ts
View File

@ -16,7 +16,7 @@ import {
} from '@inovua/reactdatagrid-community/types';
import { VALIDATION_STEP } from '../../constants/BulkImport.constant';
import { CSVImportResult } from '../../generated/type/csvImportResult';
import { CSVImportJobType } from '../BulkImport/BulkEntityImport.interface';
import { CSVImportJobType } from '../../pages/EntityImport/BulkEntityImportPage/BulkEntityImportPage.interface';
import { TitleBreadcrumbProps } from '../common/TitleBreadcrumb/TitleBreadcrumb.interface';
export interface BulkEditEntityProps {

544
openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.component.tsx
View File

@ -1,544 +0,0 @@
/*
* Copyright 2024 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import ReactDataGrid from '@inovua/reactdatagrid-community';
import '@inovua/reactdatagrid-community/index.css';
import {
TypeColumn,
TypeComputedProps,
} from '@inovua/reactdatagrid-community/types';
import { Button, Card, Col, Row, Space, Typography } from 'antd';
import { AxiosError } from 'axios';
import React, {
MutableRefObject,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from 'react';
import { useTranslation } from 'react-i18next';
import { usePapaParse } from 'react-papaparse';
import { capitalize } from 'lodash';
import {
ENTITY_IMPORT_STEPS,
VALIDATION_STEP,
} from '../../constants/BulkImport.constant';
import { SOCKET_EVENTS } from '../../constants/constants';
import { useWebSocketConnector } from '../../context/WebSocketProvider/WebSocketProvider';
import { CSVImportResult } from '../../generated/type/csvImportResult';
import {
getCSVStringFromColumnsAndDataSource,
getEntityColumnsAndDataSourceFromCSV,
} from '../../utils/CSV/CSV.utils';
import csvUtilsClassBase from '../../utils/CSV/CSVUtilsClassBase';
import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils';
import Banner from '../common/Banner/Banner';
import { ImportStatus } from '../common/EntityImport/ImportStatus/ImportStatus.component';
import Stepper from '../Settings/Services/Ingestion/IngestionStepper/IngestionStepper.component';
import { UploadFile } from '../UploadFile/UploadFile';
import './bulk-entity-import.style.less';
import {
BulkImportProps,
CSVImportAsyncWebsocketResponse,
CSVImportJobType,
} from './BulkEntityImport.interface';
let inEdit = false;
const BulkEntityImport = ({
entityType,
fqn,
onValidateCsvString,
onSuccess,
}: BulkImportProps) => {
const { socket } = useWebSocketConnector();
const [activeAsyncImportJob, setActiveAsyncImportJob] =
useState<CSVImportJobType>();
const activeAsyncImportJobRef = useRef<CSVImportJobType>();
const [activeStep, setActiveStep] = useState<VALIDATION_STEP>(
VALIDATION_STEP.UPLOAD
);
const activeStepRef = useRef<VALIDATION_STEP>(VALIDATION_STEP.UPLOAD);
const { t } = useTranslation();
const [isValidating, setIsValidating] = useState(false);
const [validationData, setValidationData] = useState<CSVImportResult>();
const [columns, setColumns] = useState<TypeColumn[]>([]);
const [dataSource, setDataSource] = useState<Record<string, string>[]>([]);
const { readString } = usePapaParse();
const [validateCSVData, setValidateCSVData] =
useState<{ columns: TypeColumn[]; dataSource: Record<string, string>[] }>();
const [gridRef, setGridRef] = useState<
MutableRefObject<TypeComputedProps | null>
>({ current: null });
const filterColumns = useMemo(
() =>
columns?.filter(
(col) =>
!csvUtilsClassBase.hideImportsColumnList().includes(col.name ?? '')
),
[columns]
);
const focusToGrid = useCallback(() => {
setGridRef((ref) => {
ref.current?.focus();
return ref;
});
}, [setGridRef]);
const handleActiveStepChange = useCallback(
(step: VALIDATION_STEP) => {
setActiveStep(step);
activeStepRef.current = step;
},
[setActiveStep, activeStepRef]
);
const onCSVReadComplete = useCallback(
(results: { data: string[][] }) => {
// results.data is returning data with unknown type
const { columns, dataSource } = getEntityColumnsAndDataSourceFromCSV(
results.data as string[][],
entityType
);
setDataSource(dataSource);
setColumns(columns);
handleActiveStepChange(VALIDATION_STEP.EDIT_VALIDATE);
setTimeout(focusToGrid, 500);
},
[entityType, setDataSource, setColumns, handleActiveStepChange, focusToGrid]
);
const handleLoadData = useCallback(
async (e: ProgressEvent<FileReader>) => {
try {
const result = e.target?.result as string;
const validationResponse = await onValidateCsvString(result, true);
const jobData: CSVImportJobType = {
...validationResponse,
type: 'initialLoad',
initialResult: result,
};
setActiveAsyncImportJob(jobData);
activeAsyncImportJobRef.current = jobData;
} catch (error) {
showErrorToast(error as AxiosError);
}
},
[onCSVReadComplete]
);
const onEditComplete = useCallback(
({ value, columnId, rowId }) => {
const data = [...dataSource];
data[rowId][columnId] = value;
setDataSource(data);
},
[dataSource]
);
const handleBack = () => {
if (activeStep === VALIDATION_STEP.UPDATE) {
handleActiveStepChange(VALIDATION_STEP.EDIT_VALIDATE);
} else {
handleActiveStepChange(VALIDATION_STEP.UPLOAD);
}
};
const handleValidate = async () => {
setIsValidating(true);
setValidateCSVData(undefined);
try {
// Call the validate API
const csvData = getCSVStringFromColumnsAndDataSource(columns, dataSource);
const response = await onValidateCsvString(
csvData,
activeStep === VALIDATION_STEP.EDIT_VALIDATE
);
const jobData: CSVImportJobType = {
...response,
type: 'onValidate',
};
setActiveAsyncImportJob(jobData);
activeAsyncImportJobRef.current = jobData;
} catch (error) {
showErrorToast(error as AxiosError);
setIsValidating(false);
}
};
const onEditStart = () => {
inEdit = true;
};
const onEditStop = () => {
requestAnimationFrame(() => {
inEdit = false;
gridRef.current?.focus();
});
};
const onKeyDown = (event: KeyboardEvent) => {
if (inEdit) {
if (event.key === 'Escape') {
const [rowIndex, colIndex] = gridRef.current?.computedActiveCell ?? [
0, 0,
];
const column = gridRef.current?.getColumnBy(colIndex);
gridRef.current?.cancelEdit?.({
rowIndex,
columnId: column?.name ?? '',
});
}
return;
}
const grid = gridRef.current;
if (!grid) {
return;
}
let [rowIndex, colIndex] = grid.computedActiveCell ?? [0, 0];
if (event.key === ' ' || event.key === 'Enter') {
const column = grid.getColumnBy(colIndex);
grid.startEdit?.({ columnId: column.name ?? '', rowIndex });
event.preventDefault();
return;
}
if (event.key !== 'Tab') {
return;
}
event.preventDefault();
event.stopPropagation();
const direction = event.shiftKey ? -1 : 1;
const columns = grid.visibleColumns;
const rowCount = grid.count;
colIndex += direction;
if (colIndex === -1) {
colIndex = columns.length - 1;
rowIndex -= 1;
}
if (colIndex === columns.length) {
rowIndex += 1;
colIndex = 0;
}
if (rowIndex < 0 || rowIndex === rowCount) {
return;
}
grid?.setActiveCell([rowIndex, colIndex]);
};
const handleAddRow = useCallback(() => {
setDataSource((data) => {
setTimeout(() => {
gridRef.current?.scrollToId(data.length + '');
gridRef.current?.focus();
}, 1);
return [...data, { id: data.length + '' }];
});
}, [gridRef]);
const handleRetryCsvUpload = () => {
setValidationData(undefined);
handleActiveStepChange(VALIDATION_STEP.UPLOAD);
};
const handleResetImportJob = useCallback(() => {
setActiveAsyncImportJob(undefined);
activeAsyncImportJobRef.current = undefined;
}, [setActiveAsyncImportJob, activeAsyncImportJobRef]);
const handleImportWebsocketResponseWithActiveStep = useCallback(
(importResults: CSVImportResult) => {
const activeStep = activeStepRef.current;
if (activeStep === VALIDATION_STEP.UPDATE) {
if (importResults?.status === 'failure') {
setValidationData(importResults);
readString(importResults?.importResultsCsv ?? '', {
worker: true,
skipEmptyLines: true,
complete: (results) => {
// results.data is returning data with unknown type
setValidateCSVData(
getEntityColumnsAndDataSourceFromCSV(
results.data as string[][],
entityType
)
);
},
});
handleActiveStepChange(VALIDATION_STEP.UPDATE);
setIsValidating(false);
} else {
showSuccessToast(
t('message.entity-details-updated', {
entityType: capitalize(entityType),
fqn,
})
);
onSuccess();
handleResetImportJob();
setIsValidating(false);
}
} else if (activeStep === VALIDATION_STEP.EDIT_VALIDATE) {
setValidationData(importResults);
handleActiveStepChange(VALIDATION_STEP.UPDATE);
readString(importResults?.importResultsCsv ?? '', {
worker: true,
skipEmptyLines: true,
complete: (results) => {
// results.data is returning data with unknown type
setValidateCSVData(
getEntityColumnsAndDataSourceFromCSV(
results.data as string[][],
entityType
)
);
},
});
handleResetImportJob();
setIsValidating(false);
}
},
[
activeStepRef,
entityType,
fqn,
onSuccess,
handleResetImportJob,
handleActiveStepChange,
]
);
const handleImportWebsocketResponse = useCallback(
(websocketResponse: CSVImportAsyncWebsocketResponse) => {
if (!websocketResponse.jobId) {
return;
}
const activeImportJob = activeAsyncImportJobRef.current;
if (websocketResponse.jobId === activeImportJob?.jobId) {
setActiveAsyncImportJob((job) => {
if (!job) {
return;
}
return {
...job,
...websocketResponse,
};
});
if (websocketResponse.status === 'COMPLETED') {
const importResults = websocketResponse.result;
// If the job is complete and the status is either failure or aborted
// then reset the validation data and active step
if (['failure', 'aborted'].includes(importResults?.status ?? '')) {
setValidationData(importResults);
handleActiveStepChange(VALIDATION_STEP.UPLOAD);
handleResetImportJob();
return;
}
// If the job is complete and the status is success
// and job was for initial load then check if the initial result is available
// and then read the initial result
if (
activeImportJob.type === 'initialLoad' &&
activeImportJob.initialResult
) {
readString(activeImportJob.initialResult, {
worker: true,
skipEmptyLines: true,
complete: onCSVReadComplete,
});
handleResetImportJob();
return;
}
handleImportWebsocketResponseWithActiveStep(importResults);
}
}
},
[
activeStepRef,
activeAsyncImportJobRef,
onCSVReadComplete,
setActiveAsyncImportJob,
handleResetImportJob,
handleActiveStepChange,
]
);
useEffect(() => {
if (socket) {
socket.on(SOCKET_EVENTS.CSV_IMPORT_CHANNEL, (importResponse) => {
if (importResponse) {
const importResponseData = JSON.parse(
importResponse
) as CSVImportAsyncWebsocketResponse;
handleImportWebsocketResponse(importResponseData);
}
});
}
return () => {
socket && socket.off(SOCKET_EVENTS.CSV_IMPORT_CHANNEL);
};
}, [socket]);
return (
<Row gutter={[16, 16]}>
<Col span={24}>
<Stepper activeStep={activeStep} steps={ENTITY_IMPORT_STEPS} />
</Col>
{activeAsyncImportJob?.jobId && (
<Col span={24}>
<Banner
className="border-radius"
isLoading={!activeAsyncImportJob.error}
message={
activeAsyncImportJob.error ?? activeAsyncImportJob.message ?? ''
}
type={activeAsyncImportJob.error ? 'error' : 'success'}
/>
</Col>
)}
<Col span={24}>
{activeStep === 0 && (
<>
{validationData?.abortReason ? (
<Card className="m-t-lg">
<Space
align="center"
className="w-full justify-center p-lg text-center"
direction="vertical"
size={16}>
<Typography.Text
className="text-center"
data-testid="abort-reason">
<strong className="d-block">{t('label.aborted')}</strong>{' '}
{validationData.abortReason}
</Typography.Text>
<Space size={16}>
<Button
ghost
data-testid="cancel-button"
type="primary"
onClick={handleRetryCsvUpload}>
{t('label.back')}
</Button>
</Space>
</Space>
</Card>
) : (
<UploadFile fileType=".csv" onCSVUploaded={handleLoadData} />
)}
</>
)}
{activeStep === 1 && (
<ReactDataGrid
editable
columns={filterColumns}
dataSource={dataSource}
defaultActiveCell={[0, 0]}
handle={setGridRef}
idProperty="id"
loading={isValidating}
minRowHeight={30}
showZebraRows={false}
style={{ height: 'calc(100vh - 245px)' }}
onEditComplete={onEditComplete}
onEditStart={onEditStart}
onEditStop={onEditStop}
onKeyDown={onKeyDown}
/>
)}
{activeStep === 2 && validationData && (
<Row gutter={[16, 16]}>
<Col span={24}>
<ImportStatus csvImportResult={validationData} />
</Col>
<Col span={24}>
{validateCSVData && (
<ReactDataGrid
idProperty="id"
loading={isValidating}
style={{ height: 'calc(100vh - 300px)' }}
{...validateCSVData}
/>
)}
</Col>
</Row>
)}
</Col>
{activeStep > 0 && (
<Col span={24}>
{activeStep === 1 && (
<Button data-testid="add-row-btn" onClick={handleAddRow}>
{`+ ${t('label.add-row')}`}
</Button>
)}
<div className="float-right import-footer">
{activeStep > 0 && (
<Button disabled={isValidating} onClick={handleBack}>
{t('label.previous')}
</Button>
)}
{activeStep < 3 && (
<Button
className="m-l-sm"
disabled={isValidating}
type="primary"
onClick={handleValidate}>
{activeStep === 2 ? t('label.update') : t('label.next')}
</Button>
)}
</div>
</Col>
)}
</Row>
);
};
export default BulkEntityImport;

18
openmetadata-ui/src/main/resources/ui/src/components/BulkImport/bulk-entity-import.style.less
View File

@ -1,18 +0,0 @@
/*
* Copyright 2024 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
.reserve-right-sidebar {
.import-footer {
// Right side padding 20 + 64 width of sidebar
padding-right: 84px;
}
}

15
openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.component.tsx
View File

@ -41,7 +41,7 @@ import { DE_ACTIVE_COLOR } from '../../../constants/constants';
import { ExportTypes } from '../../../constants/Export.constants';
import { usePermissionProvider } from '../../../context/PermissionProvider/PermissionProvider';
import { ResourceEntity } from '../../../context/PermissionProvider/PermissionProvider.interface';
import { EntityAction, EntityType } from '../../../enums/entity.enum';
import { EntityType } from '../../../enums/entity.enum';
import { Glossary } from '../../../generated/entity/data/glossary';
import {
GlossaryTerm,
@ -58,12 +58,14 @@ import {
patchGlossaryTerm,
} from '../../../rest/glossaryAPI';
import { getEntityDeleteMessage } from '../../../utils/CommonUtils';
import { getEntityVoteStatus } from '../../../utils/EntityUtils';
import {
getEntityImportPath,
getEntityVoteStatus,
} from '../../../utils/EntityUtils';
import Fqn from '../../../utils/Fqn';
import { checkPermission } from '../../../utils/PermissionsUtils';
import {
getGlossaryPath,
getGlossaryPathWithAction,
getGlossaryTermsVersionsPath,
getGlossaryVersionsPath,
} from '../../../utils/RouterUtils';
@ -211,12 +213,7 @@ const GlossaryHeader = ({
}, [fqn]);
const handleGlossaryImport = () =>
history.push(
getGlossaryPathWithAction(
selectedData.fullyQualifiedName ?? '',
EntityAction.IMPORT
)
);
history.push(getEntityImportPath(EntityType.GLOSSARY_TERM, fqn));
const handleVersionClick = async () => {
let path: string;

12
openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryV1.component.tsx
View File

@ -14,7 +14,7 @@
import { AxiosError } from 'axios';
import { compare } from 'fast-json-patch';
import { cloneDeep, isEmpty } from 'lodash';
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import React, { useCallback, useEffect, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useHistory, useParams } from 'react-router-dom';
import { withActivityFeed } from '../../components/AppRouter/withActivityFeed';
@ -49,7 +49,6 @@ import GlossaryTermModal from './GlossaryTermModal/GlossaryTermModal.component';
import GlossaryTermsV1 from './GlossaryTerms/GlossaryTermsV1.component';
import { GlossaryV1Props } from './GlossaryV1.interfaces';
import './glossaryV1.less';
import ImportGlossary from './ImportGlossary/ImportGlossary';
import { ModifiedGlossary, useGlossaryStore } from './useGlossary.store';
const GlossaryV1 = ({
@ -101,11 +100,6 @@ const GlossaryV1 = ({
const { id, fullyQualifiedName } = activeGlossary ?? {};
const isImportAction = useMemo(
() => action === EntityAction.IMPORT,
[action]
);
const fetchGlossaryTerm = async (
params?: ListGlossaryTermsParams,
refresh?: boolean
@ -336,9 +330,7 @@ const GlossaryV1 = ({
setIsTabExpanded(!isTabExpanded);
};
return isImportAction ? (
<ImportGlossary glossaryName={selectedData.fullyQualifiedName ?? ''} />
) : (
return (
<>
{(isLoading || isPermissionLoading) && <Loader />}

28
openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryV1.test.tsx
View File

@ -11,13 +11,7 @@
* limitations under the License.
*/
import {
act,
findByText,
getByTestId,
queryByText,
render,
} from '@testing-library/react';
import { findByText, queryByText, render } from '@testing-library/react';
import React from 'react';
import {
mockedGlossaries,
@ -26,7 +20,7 @@ import {
import GlossaryV1 from './GlossaryV1.component';
import { GlossaryV1Props } from './GlossaryV1.interfaces';
let params = {
const params = {
glossaryName: 'GlossaryName',
action: '',
};
@ -115,12 +109,6 @@ jest.mock('../ActivityFeed/FeedEditor/FeedEditor', () => {
return jest.fn().mockReturnValue(<p>FeedEditor</p>);
});
jest.mock('./ImportGlossary/ImportGlossary', () =>
jest
.fn()
.mockReturnValue(<div data-testid="import-glossary">ImportGlossary</div>)
);
jest.mock('../../components/AppRouter/withActivityFeed', () => ({
withActivityFeed: jest.fn().mockImplementation((component) => component),
}));
@ -185,16 +173,4 @@ describe('Test Glossary component', () => {
expect(glossaryTerm).toBeInTheDocument();
expect(glossaryDetails).not.toBeInTheDocument();
});
it('Should render import glossary component', async () => {
params = { ...params, action: 'import' };
await act(async () => {
const { container } = render(<GlossaryV1 {...mockProps} />);
const importGlossary = getByTestId(container, 'import-glossary');
expect(importGlossary).toBeInTheDocument();
});
});
});

124
openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.test.tsx
View File

@ -1,124 +0,0 @@
/*
* Copyright 2023 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { act, fireEvent, render, screen } from '@testing-library/react';
import React from 'react';
import { CSVImportResult } from '../../../generated/type/csvImportResult';
import { importGlossaryInCSVFormat } from '../../../rest/glossaryAPI';
import ImportGlossary from './ImportGlossary';
const mockPush = jest.fn();
const glossaryName = 'Glossary1';
const mockCsvImportResult = {
dryRun: true,
status: 'success',
numberOfRowsProcessed: 3,
numberOfRowsPassed: 3,
numberOfRowsFailed: 0,
importResultsCsv: `status,details,parent,name*,displayName,description,synonyms,relatedTerms,references,tags\r
success,Entity created,,Glossary2 Term,Glossary2 Term displayName,Description for Glossary2 Term,,,,\r
success,Entity created,,Glossary2 term2,Glossary2 term2,Description data.,,,,\r`,
} as CSVImportResult;
jest.mock('../../common/TitleBreadcrumb/TitleBreadcrumb.component', () =>
jest.fn().mockReturnValue(<div data-testid="breadcrumb">Breadcrumb</div>)
);
jest.mock('../../common/Loader/Loader', () =>
jest.fn().mockReturnValue(<div data-testid="loader">Loader</div>)
);
jest.mock('../../PageLayoutV1/PageLayoutV1', () => {
return jest.fn().mockImplementation(({ children }) => <p>{children}</p>);
});
jest.mock('../../BulkImport/BulkEntityImport.component', () =>
jest.fn().mockImplementation(({ onSuccess, onValidateCsvString }) => (
<div>
<button onClick={onSuccess}>SuccessButton</button>
<button
onClick={() => onValidateCsvString('markdown: This is test', true)}>
ValidateCsvButton
</button>
<p>BulkEntityImport</p>
</div>
))
);
jest.mock('../../../rest/glossaryAPI', () => ({
importGlossaryInCSVFormat: jest
.fn()
.mockImplementation(() => Promise.resolve(mockCsvImportResult)),
}));
jest.mock('react-router-dom', () => ({
useHistory: jest.fn().mockImplementation(() => ({
push: mockPush,
})),
}));
jest.mock('../../../utils/RouterUtils', () => ({
getGlossaryPath: jest.fn().mockImplementation((fqn) => `/glossary/${fqn}`),
}));
jest.mock('../../../utils/ToastUtils', () => ({
showErrorToast: jest.fn(),
}));
jest.mock('../../common/EntityImport/EntityImport.component', () => ({
EntityImport: jest.fn().mockImplementation(({ children, onImport }) => {
return (
<div data-testid="entity-import">
{children}{' '}
<button data-testid="import" onClick={onImport}>
import
</button>
</div>
);
}),
}));
describe('Import Glossary', () => {
it('Should render the all components', async () => {
render(<ImportGlossary glossaryName={glossaryName} />);
expect(await screen.findByTestId('breadcrumb')).toBeInTheDocument();
expect(screen.getByText('BulkEntityImport')).toBeInTheDocument();
expect(screen.getByText('SuccessButton')).toBeInTheDocument();
expect(screen.getByText('ValidateCsvButton')).toBeInTheDocument();
});
it('should redirect the page when onSuccess get triggered', async () => {
render(<ImportGlossary glossaryName={glossaryName} />);
const successButton = screen.getByText('SuccessButton');
await act(async () => {
fireEvent.click(successButton);
});
expect(mockPush).toHaveBeenCalled();
});
it('should call the importGlossaryInCSVFormat api when validate props is trigger', async () => {
render(<ImportGlossary glossaryName={glossaryName} />);
const successButton = screen.getByText('ValidateCsvButton');
await act(async () => {
fireEvent.click(successButton);
});
expect(importGlossaryInCSVFormat).toHaveBeenCalledWith(
'Glossary1',
'markdown: This is test',
true
);
});
});

93
openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/ImportGlossary.tsx
View File

@ -1,93 +0,0 @@
/*
* Copyright 2023 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Col, Row } from 'antd';
import { AxiosError } from 'axios';
import React, { FC, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { useHistory } from 'react-router-dom';
import { EntityType } from '../../../enums/entity.enum';
import { importGlossaryInCSVFormat } from '../../../rest/glossaryAPI';
import { getGlossaryPath } from '../../../utils/RouterUtils';
import { showErrorToast } from '../../../utils/ToastUtils';
import BulkEntityImport from '../../BulkImport/BulkEntityImport.component';
import TitleBreadcrumb from '../../common/TitleBreadcrumb/TitleBreadcrumb.component';
import { TitleBreadcrumbProps } from '../../common/TitleBreadcrumb/TitleBreadcrumb.interface';
import PageLayoutV1 from '../../PageLayoutV1/PageLayoutV1';
import './import-glossary.less';
interface Props {
glossaryName: string;
}
const ImportGlossary: FC<Props> = ({ glossaryName }) => {
const { t } = useTranslation();
const history = useHistory();
const breadcrumbList: TitleBreadcrumbProps['titleLinks'] = useMemo(
() => [
{
name: t('label.glossary-plural'),
url: getGlossaryPath(),
activeTitle: false,
},
{
name: glossaryName,
url: getGlossaryPath(glossaryName),
},
],
[glossaryName]
);
const handleGlossaryRedirection = () => {
history.push(getGlossaryPath(glossaryName));
};
const handleImportCsv = async (data: string, dryRun = true) => {
try {
const response = await importGlossaryInCSVFormat(
glossaryName,
data,
dryRun
);
return response;
} catch (error) {
showErrorToast(error as AxiosError);
return;
}
};
return (
<PageLayoutV1
pageTitle={t('label.import-entity', {
entity: t('label.glossary-term-plural'),
})}>
<Row className="import-glossary p-x-lg" gutter={[16, 16]}>
<Col span={24}>
<TitleBreadcrumb titleLinks={breadcrumbList} />
</Col>
<Col span={24}>
<BulkEntityImport
entityType={EntityType.GLOSSARY_TERM}
fqn={glossaryName}
onSuccess={handleGlossaryRedirection}
onValidateCsvString={handleImportCsv}
/>
</Col>
</Row>
</PageLayoutV1>
);
};
export default ImportGlossary;

52
openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportGlossary/import-glossary.less
View File

@ -1,52 +0,0 @@
/*
* Copyright 2023 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@bg-white: #ffffff;
@border-color: #6b728066;
@check-mark-color: #28a744;
@fail-color: #cb2531;
.ant-upload.file-dragger-wrapper {
border-color: @border-color;
border-width: 2px;
&:not(.ant-upload-disabled) {
&:hover {
border-color: @border-color;
}
}
background: @bg-white;
height: 360px;
width: unset;
}
.glossary-preview-footer {
box-shadow: 1px -2px 4px rgba(0, 0, 0, 0.12);
position: fixed;
bottom: 0;
right: 0;
left: 0;
z-index: 4;
}
.import-glossary {
.browse-text {
border-bottom: 1px solid;
}
}
.passed-row {
color: @check-mark-color;
}
.failed-row {
color: @fail-color;
}

4
openmetadata-ui/src/main/resources/ui/src/components/common/EntityImport/EntityImport.component.tsx
View File

@ -29,11 +29,11 @@ import {
CSVImportResult,
Status,
} from '../../../generated/type/csvImportResult';
import { showErrorToast } from '../../../utils/ToastUtils';
import {
CSVImportAsyncWebsocketResponse,
CSVImportJobType,
} from '../../BulkImport/BulkEntityImport.interface';
} from '../../../pages/EntityImport/BulkEntityImportPage/BulkEntityImportPage.interface';
import { showErrorToast } from '../../../utils/ToastUtils';
import Stepper from '../../Settings/Services/Ingestion/IngestionStepper/IngestionStepper.component';
import { UploadFile } from '../../UploadFile/UploadFile';
import Banner from '../Banner/Banner';

2
openmetadata-ui/src/main/resources/ui/src/components/common/EntityImport/EntityImport.interface.ts
View File

@ -12,7 +12,7 @@
*/
import React from 'react';
import { CSVImportResult } from '../../../generated/type/csvImportResult';
import { CSVImportAsyncResponse } from '../../BulkImport/BulkEntityImport.interface';
import { CSVImportAsyncResponse } from '../../../pages/EntityImport/BulkEntityImportPage/BulkEntityImportPage.interface';
export interface EntityImportProps {
entityName: string;

1
openmetadata-ui/src/main/resources/ui/src/constants/BulkImport.constant.ts
View File

@ -20,6 +20,7 @@ export const SUPPORTED_BULK_IMPORT_EDIT_ENTITY = [
ResourceEntity.DATABASE_SERVICE,
ResourceEntity.DATABASE,
ResourceEntity.DATABASE_SCHEMA,
ResourceEntity.GLOSSARY_TERM,
];
export enum VALIDATION_STEP {

15
openmetadata-ui/src/main/resources/ui/src/components/BulkImport/BulkEntityImport.interface.ts → openmetadata-ui/src/main/resources/ui/src/pages/EntityImport/BulkEntityImportPage/BulkEntityImportPage.interface.ts
View File

@ -1,5 +1,5 @@
/*
* Copyright 2024 Collate.
* Copyright 2025 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
@ -10,18 +10,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityType } from '../../enums/entity.enum';
import { CSVImportResult } from '../../generated/type/csvImportResult';
export interface BulkImportProps {
entityType: EntityType;
fqn: string;
onValidateCsvString: (
data: string,
dryRun?: boolean
) => Promise<CSVImportAsyncResponse | undefined>;
onSuccess: () => void;
}
import { CSVImportResult } from '../../../generated/type/csvImportResult';
export type CSVImportAsyncResponse = {
jobId: string;

36
openmetadata-ui/src/main/resources/ui/src/pages/EntityImport/BulkEntityImportPage/BulkEntityImportPage.tsx
View File

@ -18,7 +18,7 @@ import {
} from '@inovua/reactdatagrid-community/types';
import { Button, Card, Col, Row, Space, Typography } from 'antd';
import { AxiosError } from 'axios';
import { isEmpty } from 'lodash';
import { capitalize, isEmpty } from 'lodash';
import React, {
MutableRefObject,
useCallback,
@ -31,10 +31,6 @@ import { useTranslation } from 'react-i18next';
import { usePapaParse } from 'react-papaparse';
import { useHistory, useLocation, useParams } from 'react-router-dom';
import BulkEditEntity from '../../../components/BulkEditEntity/BulkEditEntity.component';
import {
CSVImportAsyncWebsocketResponse,
CSVImportJobType,
} from '../../../components/BulkImport/BulkEntityImport.interface';
import Banner from '../../../components/common/Banner/Banner';
import { ImportStatus } from '../../../components/common/EntityImport/ImportStatus/ImportStatus.component';
import TitleBreadcrumb from '../../../components/common/TitleBreadcrumb/TitleBreadcrumb.component';
@ -52,23 +48,25 @@ import { useWebSocketConnector } from '../../../context/WebSocketProvider/WebSoc
import { EntityType } from '../../../enums/entity.enum';
import { CSVImportResult } from '../../../generated/type/csvImportResult';
import { useFqn } from '../../../hooks/useFqn';
import {
importEntityInCSVFormat,
importServiceInCSVFormat,
} from '../../../rest/importExportAPI';
import {
getCSVStringFromColumnsAndDataSource,
getEntityColumnsAndDataSourceFromCSV,
} from '../../../utils/CSV/CSV.utils';
import csvUtilsClassBase from '../../../utils/CSV/CSVUtilsClassBase';
import { isBulkEditRoute } from '../../../utils/EntityBulkEdit/EntityBulkEditUtils';
import {
getBulkEntityBreadcrumbList,
getImportedEntityType,
getImportValidateAPIEntityType,
validateCsvString,
} from '../../../utils/EntityImport/EntityImportUtils';
import entityUtilClassBase from '../../../utils/EntityUtilClassBase';
import { showErrorToast, showSuccessToast } from '../../../utils/ToastUtils';
import './bulk-entity-import-page.less';
import {
CSVImportAsyncWebsocketResponse,
CSVImportJobType,
} from './BulkEntityImportPage.interface';
let inEdit = false;
@ -100,6 +98,15 @@ const BulkEntityImportPage = () => {
>({ current: null });
const [entity, setEntity] = useState<DataAssetsHeaderProps['dataAsset']>();
const filterColumns = useMemo(
() =>
columns?.filter(
(col) =>
!csvUtilsClassBase.hideImportsColumnList().includes(col.name ?? '')
),
[columns]
);
const fetchEntityData = useCallback(async () => {
try {
const response = await entityUtilClassBase.getEntityByFqn(
@ -107,7 +114,7 @@ const BulkEntityImportPage = () => {
fqn
);
setEntity(response);
} catch (error) {
} catch {
// not show error here
}
}, [entityType, fqn]);
@ -216,10 +223,7 @@ const BulkEntityImportPage = () => {
// Call the validate API
const csvData = getCSVStringFromColumnsAndDataSource(columns, dataSource);
const api =
entityType === EntityType.DATABASE_SERVICE
? importServiceInCSVFormat
: importEntityInCSVFormat;
const api = getImportValidateAPIEntityType(entityType);
const response = await api({
entityType,
@ -351,7 +355,7 @@ const BulkEntityImportPage = () => {
} else {
showSuccessToast(
t('message.entity-details-updated', {
entityType,
entityType: capitalize(entityType),
fqn,
})
);
@ -572,7 +576,7 @@ const BulkEntityImportPage = () => {
{activeStep === 1 && (
<ReactDataGrid
editable
columns={columns}
columns={filterColumns}
dataSource={dataSource}
defaultActiveCell={[0, 0]}
handle={setGridRef}

20
openmetadata-ui/src/main/resources/ui/src/rest/databaseAPI.ts
View File

@ -256,10 +256,12 @@ export const exportDatabaseDetailsInCSV = async (
recursive?: boolean;
}
) => {
// FQN should be encoded already and we should not encode the fqn here to avoid double encoding
const res = await APIClient.get(`databases/name/${fqn}/exportAsync`, {
params,
});
const res = await APIClient.get(
`databases/name/${getEncodedFqn(fqn)}/exportAsync`,
{
params,
}
);
return res.data;
};
@ -287,10 +289,12 @@ export const exportDatabaseSchemaDetailsInCSV = async (
recursive?: boolean;
}
) => {
// FQN should be encoded already and we should not encode the fqn here to avoid double encoding
const res = await APIClient.get(`databaseSchemas/name/${fqn}/exportAsync`, {
params,
});
const res = await APIClient.get(
`databaseSchemas/name/${getEncodedFqn(fqn)}/exportAsync`,
{
params,
}
);
return res.data;
};

23
openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts
View File

@ -14,7 +14,6 @@
import { AxiosResponse } from 'axios';
import { Operation } from 'fast-json-patch';
import { PagingResponse } from 'Models';
import { CSVImportAsyncResponse } from '../components/BulkImport/BulkEntityImport.interface';
import { CSVExportResponse } from '../components/Entity/EntityExportModalProvider/EntityExportModalProvider.interface';
import { VotingDataProps } from '../components/Entity/Voting/voting.interface';
import { ES_MAX_PAGE_SIZE, PAGE_SIZE_MEDIUM } from '../constants/constants';
@ -172,28 +171,6 @@ export const exportGlossaryInCSVFormat = async (glossaryName: string) => {
return response.data;
};
export const importGlossaryInCSVFormat = async (
glossaryName: string,
data: string,
dryRun = true
) => {
const configOptions = {
headers: { 'Content-type': 'text/plain' },
};
const response = await APIClient.put<
string,
AxiosResponse<CSVImportAsyncResponse>
>(
`/glossaries/name/${getEncodedFqn(
glossaryName
)}/importAsync?dryRun=${dryRun}`,
data,
configOptions
);
return response.data;
};
export const getGlossaryVersionsList = async (id: string) => {
const url = `/glossaries/${id}/versions`;

22
openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.ts
View File

@ -11,8 +11,8 @@
* limitations under the License.
*/
import { AxiosResponse } from 'axios';
import { CSVImportAsyncResponse } from '../components/BulkImport/BulkEntityImport.interface';
import { EntityType } from '../enums/entity.enum';
import { CSVImportAsyncResponse } from '../pages/EntityImport/BulkEntityImportPage/BulkEntityImportPage.interface';
import { getEncodedFqn } from '../utils/StringsUtils';
import APIClient from './index';
@ -71,3 +71,23 @@ export const importServiceInCSVFormat = async ({
return res.data;
};
export const importGlossaryInCSVFormat = async ({
name,
data,
dryRun = true,
}: importEntityInCSVFormatRequestParams) => {
const configOptions = {
headers: { 'Content-type': 'text/plain' },
};
const response = await APIClient.put<
string,
AxiosResponse<CSVImportAsyncResponse>
>(
`/glossaries/name/${getEncodedFqn(name)}/importAsync?dryRun=${dryRun}`,
data,
configOptions
);
return response.data;
};

3
openmetadata-ui/src/main/resources/ui/src/rest/serviceAPI.ts
View File

@ -193,8 +193,7 @@ export const exportDatabaseServiceDetailsInCSV = async (
}
) => {
const res = await APIClient.get(
// FQN should be encoded already and we should not encode the fqn here to avoid double encoding
`services/databaseServices/name/${fqn}/exportAsync`,
`services/databaseServices/name/${getEncodedFqn(fqn)}/exportAsync`,
{
params,
}

10
openmetadata-ui/src/main/resources/ui/src/rest/tableAPI.ts
View File

@ -239,10 +239,12 @@ export const exportTableDetailsInCSV = async (
recursive?: boolean;
}
) => {
// FQN should be encoded already and we should not encode the fqn here to avoid double encoding
const res = await APIClient.get(`tables/name/${fqn}/exportAsync`, {
params,
});
const res = await APIClient.get(
`tables/name/${getEncodedFqn(fqn)}/exportAsync`,
{
params,
}
);
return res.data;
};

2
openmetadata-ui/src/main/resources/ui/src/rest/teamsAPI.ts
View File

@ -14,13 +14,13 @@
import { AxiosResponse } from 'axios';
import { Operation } from 'fast-json-patch';
import { PagingResponse, RestoreRequestType } from 'Models';
import { CSVImportAsyncResponse } from '../components/BulkImport/BulkEntityImport.interface';
import { CSVExportResponse } from '../components/Entity/EntityExportModalProvider/EntityExportModalProvider.interface';
import { CreateTeam } from '../generated/api/teams/createTeam';
import { EntityReference } from '../generated/entity/data/table';
import { Team } from '../generated/entity/teams/team';
import { TeamHierarchy } from '../generated/entity/teams/teamHierarchy';
import { ListParams } from '../interface/API.interface';
import { CSVImportAsyncResponse } from '../pages/EntityImport/BulkEntityImportPage/BulkEntityImportPage.interface';
import { getEncodedFqn } from '../utils/StringsUtils';
import APIClient from './index';

36
openmetadata-ui/src/main/resources/ui/src/utils/EntityImport/EntityImportUtils.ts
View File

@ -16,10 +16,12 @@ import { DataAssetsHeaderProps } from '../../components/DataAssets/DataAssetsHea
import { EntityType } from '../../enums/entity.enum';
import {
importEntityInCSVFormat,
importGlossaryInCSVFormat,
importServiceInCSVFormat,
} from '../../rest/importExportAPI';
import { getEntityBreadcrumbs } from '../EntityUtils';
import { getEntityBreadcrumbs, getEntityName } from '../EntityUtils';
import i18n from '../i18next/LocalUtil';
import { getGlossaryPath } from '../RouterUtils';
type ParsedDataType<T> = Array<T>;
@ -67,7 +69,19 @@ export const getBulkEntityBreadcrumbList = (
isBulkEdit: boolean
): TitleBreadcrumbProps['titleLinks'] => {
return [
...getEntityBreadcrumbs(entity, entityType, true),
...(entityType === EntityType.GLOSSARY_TERM
? [
{
name: i18n.t('label.glossary-plural'),
url: getGlossaryPath(),
activeTitle: false,
},
{
name: getEntityName(entity),
url: getGlossaryPath(entity.fullyQualifiedName),
},
]
: getEntityBreadcrumbs(entity, entityType, true)),
{
name: i18n.t(`label.${isBulkEdit ? 'bulk-edit' : 'import'}`),
url: '',
@ -76,16 +90,26 @@ export const getBulkEntityBreadcrumbList = (
];
};
export const getImportValidateAPIEntityType = (entityType: EntityType) => {
switch (entityType) {
case EntityType.DATABASE_SERVICE:
return importServiceInCSVFormat;
case EntityType.GLOSSARY_TERM:
return importGlossaryInCSVFormat;
default:
return importEntityInCSVFormat;
}
};
export const validateCsvString = async (
csvData: string,
entityType: EntityType,
fqn: string,
isBulkEdit: boolean
) => {
const api =
entityType === EntityType.DATABASE_SERVICE
? importServiceInCSVFormat
: importEntityInCSVFormat;
const api = getImportValidateAPIEntityType(entityType);
const response = await api({
entityType,

23
openmetadata-ui/src/main/resources/ui/src/utils/EntityUtilClassBase.ts
View File

@ -45,6 +45,7 @@ import {
getDatabaseDetailsByFQN,
getDatabaseSchemaDetailsByFQN,
} from '../rest/databaseAPI';
import { getGlossariesByName } from '../rest/glossaryAPI';
import { getServiceByFQN } from '../rest/serviceAPI';
import { getTableDetailsByFQN } from '../rest/tableAPI';
import { ExtraDatabaseDropdownOptions } from './Database/Database.util';
@ -68,7 +69,6 @@ import {
getTeamsWithFqnPath,
getUserPath,
} from './RouterUtils';
import { getEncodedFqn } from './StringsUtils';
import { ExtraTableDropdownOptions } from './TableUtils';
import { getTestSuiteDetailsPath } from './TestSuiteUtils';
@ -292,6 +292,9 @@ class EntityUtilClassBase {
return getDatabaseDetailsByFQN(fqn, { fields });
case EntityType.DATABASE_SCHEMA:
return getDatabaseSchemaDetailsByFQN(fqn, { fields });
case EntityType.GLOSSARY_TERM:
return getGlossariesByName(fqn, { fields });
default:
return getTableDetailsByFQN(fqn, { fields });
}
@ -408,29 +411,19 @@ class EntityUtilClassBase {
| APICollection
): ItemType[] {
const isEntityDeleted = _entityDetails?.deleted ?? false;
// We are encoding here since we are getting the decoded fqn from the OSS code
const encodedFqn = getEncodedFqn(_fqn);
switch (_entityType) {
case EntityType.TABLE:
return [
...ExtraTableDropdownOptions(
encodedFqn,
_permission,
isEntityDeleted
),
...ExtraTableDropdownOptions(_fqn, _permission, isEntityDeleted),
];
case EntityType.DATABASE:
return [
...ExtraDatabaseDropdownOptions(
encodedFqn,
_permission,
isEntityDeleted
),
...ExtraDatabaseDropdownOptions(_fqn, _permission, isEntityDeleted),
];
case EntityType.DATABASE_SCHEMA:
return [
...ExtraDatabaseSchemaDropdownOptions(
encodedFqn,
_fqn,
_permission,
isEntityDeleted
),
@ -438,7 +431,7 @@ class EntityUtilClassBase {
case EntityType.DATABASE_SERVICE:
return [
...ExtraDatabaseServiceDropdownOptions(
encodedFqn,
_fqn,
_permission,
isEntityDeleted
),

2
openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx
View File

@ -2504,7 +2504,7 @@ export const getEntityImportPath = (entityType: EntityType, fqn: string) => {
return ROUTES.ENTITY_IMPORT.replace(
PLACEHOLDER_ROUTE_ENTITY_TYPE,
entityType
).replace(PLACEHOLDER_ROUTE_FQN, fqn);
).replace(PLACEHOLDER_ROUTE_FQN, getEncodedFqn(fqn));
};
export const getEntityBulkEditPath = (entityType: EntityType, fqn: string) => {