yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-19 12:50:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ui): support batchSize & flushIntervalInSec while ES re-indexing (#8082)

Browse Source 
* fix(ui): support batchSize & flushIntervalInSec while ES re-indexing

* add refresh button for stream mode

* add refresh batch for fallback option

* update flush interval label
This commit is contained in:
Chirag Madlani 2022-10-12 12:16:19 +05:30 committed by GitHub
parent 6565655e11
commit 5c7d01a9df
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 383 additions and 308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
openmetadata-ui/src/main/resources/ui/src/axiosAPIs/elastic-index-API.ts
View File

@ -31,12 +31,16 @@ export const reIndexByPublisher = async ({
runMode, runMode,
entities = ['all'], entities = ['all'],
recreateIndex = true, recreateIndex = true,
batchSize,
flushIntervalInSec,
}: CreateEventPublisherJob) => { }: CreateEventPublisherJob) => {
const payload = { const payload = {
publisherType: PublisherType.ElasticSearch, publisherType: PublisherType.ElasticSearch,
runMode, runMode,
recreateIndex, recreateIndex,
entities, entities,
batchSize,
flushIntervalInSec,
}; };
const res = await axiosClient.post('/indexResource/reindex', payload); const res = await axiosClient.post('/indexResource/reindex', payload);

1
openmetadata-ui/src/main/resources/ui/src/constants/constants.ts
View File

@ -258,6 +258,7 @@ export const SOCKET_EVENTS = {
ACTIVITY_FEED: 'activityFeed', ACTIVITY_FEED: 'activityFeed',
TASK_CHANNEL: 'taskChannel', TASK_CHANNEL: 'taskChannel',
MENTION_CHANNEL: 'mentionChannel', MENTION_CHANNEL: 'mentionChannel',
JOB_STATUS: 'jobStatus',
}; };
export const IN_PAGE_SEARCH_ROUTES: Record<string, Array<string>> = { export const IN_PAGE_SEARCH_ROUTES: Record<string, Array<string>> = {

29
openmetadata-ui/src/main/resources/ui/src/constants/elasticsearch.constant.ts
View File

@ -49,3 +49,32 @@ export const ELASTIC_SEARCH_INDEX_ENTITIES = [
label: 'Tag', label: 'Tag',
}, },
]; ];
export const ELASTIC_SEARCH_INITIAL_VALUES = {
entities: [
'table',
'topic',
'dashboard',
'pipeline',
'mlmodel',
'bot',
'user',
'team',
'glossaryTerm',
'tag',
],
batchSize: 100,
flushIntervalInSec: 30,
recreateIndex: false,
};
export const RECREATE_INDEX_OPTIONS = [
{
label: 'Yes',
value: true,
},
{
label: 'No',
value: false,
},
];

102
openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/elastic-re-index-modal.component.tsx Normal file
View File

@ -0,0 +1,102 @@
/*
* Copyright 2022 Collate
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Checkbox, Col, Form, Input, Modal, Row, Select } from 'antd';
import React, { useState } from 'react';
import {
ELASTIC_SEARCH_INDEX_ENTITIES,
ELASTIC_SEARCH_INITIAL_VALUES,
RECREATE_INDEX_OPTIONS,
} from '../../constants/elasticsearch.constant';
import { CreateEventPublisherJob } from '../../generated/api/createEventPublisherJob';
interface ReIndexAllModalInterface {
visible: boolean;
onCancel: () => void;
onSave?: (data: CreateEventPublisherJob) => void;
confirmLoading: boolean;
}
const ReIndexAllModal = ({
visible,
onCancel,
onSave,
confirmLoading,
}: ReIndexAllModalInterface) => {
const [entities, setEntities] = useState<string[]>(
ELASTIC_SEARCH_INITIAL_VALUES.entities
);
return (
<Modal
centered
confirmLoading={confirmLoading}
okButtonProps={{
form: 're-index-form',
type: 'primary',
htmlType: 'submit',
}}
okText="Submit"
title="Re-Index Elastic Search"
visible={visible}
width={650}
onCancel={onCancel}>
<Form
id="re-index-form"
layout="vertical"
name="elastic-search-re-index"
onFinish={onSave}>
<Form.Item
initialValue={false}
label="Recreate indexes"
name="recreateIndex">
<Select
data-testid="re-index-selector"
options={RECREATE_INDEX_OPTIONS}
/>
</Form.Item>
<Form.Item initialValue={entities} label="Entities" name="entities">
<Checkbox.Group
onChange={(values) => setEntities(values as string[])}>
<Row gutter={[16, 16]}>
{ELASTIC_SEARCH_INDEX_ENTITIES.map((option) => (
<Col key={option.value} span={6}>
<Checkbox value={option.value}>{option.label}</Checkbox>
</Col>
))}
</Row>
</Checkbox.Group>
</Form.Item>
<Form.Item
initialValue={ELASTIC_SEARCH_INITIAL_VALUES.flushIntervalInSec}
label="Flush Interval (secs):"
name="flushIntervalInSec">
<Input
data-testid="flush-interval-in-sec"
placeholder="Enter seconds"
/>
</Form.Item>
<Form.Item
initialValue={ELASTIC_SEARCH_INITIAL_VALUES.batchSize}
label="Batch Size:"
name="batchSize">
<Input data-testid="batch-size" placeholder="Enter batch size" />
</Form.Item>
</Form>
</Modal>
);
};
export default ReIndexAllModal;

542
openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/elastic-search-index.page.tsx
View File

@ -11,20 +11,8 @@
* limitations under the License. * limitations under the License.
*/ */
import { QuestionCircleOutlined, ReloadOutlined } from '@ant-design/icons'; import { ReloadOutlined } from '@ant-design/icons';
import { import { Badge, Button, Card, Col, Divider, Row, Space } from 'antd';
Badge,
Button,
Card,
Checkbox,
Col,
Row,
Skeleton,
Space,
Switch,
Tooltip,
Typography,
} from 'antd';
import { AxiosError } from 'axios'; import { AxiosError } from 'axios';
import { isEmpty, startCase } from 'lodash'; import { isEmpty, startCase } from 'lodash';
import React, { useEffect, useState } from 'react'; import React, { useEffect, useState } from 'react';
@ -33,10 +21,11 @@ import {
reIndexByPublisher, reIndexByPublisher,
} from '../../axiosAPIs/elastic-index-API'; } from '../../axiosAPIs/elastic-index-API';
import RichTextEditorPreviewer from '../../components/common/rich-text-editor/RichTextEditorPreviewer'; import RichTextEditorPreviewer from '../../components/common/rich-text-editor/RichTextEditorPreviewer';
import { ELASTIC_SEARCH_INDEX_ENTITIES } from '../../constants/elasticsearch.constant'; import { useWebSocketConnector } from '../../components/web-scoket/web-scoket.provider';
import { SOCKET_EVENTS } from '../../constants/constants';
import { CreateEventPublisherJob } from '../../generated/api/createEventPublisherJob';
import { import {
EventPublisherJob, EventPublisherJob,
PublisherType,
RunMode, RunMode,
} from '../../generated/settings/eventPublisherJob'; } from '../../generated/settings/eventPublisherJob';
import { useAuth } from '../../hooks/authHooks'; import { useAuth } from '../../hooks/authHooks';
@ -48,6 +37,7 @@ import {
import SVGIcons from '../../utils/SvgUtils'; import SVGIcons from '../../utils/SvgUtils';
import { getDateTimeByTimeStampWithZone } from '../../utils/TimeUtils'; import { getDateTimeByTimeStampWithZone } from '../../utils/TimeUtils';
import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils'; import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils';
import ReIndexAllModal from './elastic-re-index-modal.component';
import './elastic-search-index.style.less'; import './elastic-search-index.style.less';
const ElasticSearchIndexPage = () => { const ElasticSearchIndexPage = () => {
@ -57,19 +47,11 @@ const ElasticSearchIndexPage = () => {
const { isAdminUser } = useAuth(); const { isAdminUser } = useAuth();
const [batchLoading, setBatchLoading] = useState(false); const [batchLoading, setBatchLoading] = useState(false);
const [streamLoading, setStreamLoading] = useState(false); const [streamLoading, setStreamLoading] = useState(false);
const [recreateIndex, setRecreateIndex] = useState(false); const [confirmLoading, setConfirmLoading] = useState(false);
const [entities, setEntities] = useState<string[]>([
'table', const [isModalOpen, setModalOpen] = useState(false);
'topic',
'dashboard', const { socket } = useWebSocketConnector();
'pipeline',
'mlmodel',
'bot',
'user',
'team',
'glossaryTerm',
'tag',
]);
const fetchBatchReIndexedData = async () => { const fetchBatchReIndexedData = async () => {
try { try {
@ -97,14 +79,13 @@ const ElasticSearchIndexPage = () => {
} }
}; };
const performReIndexAll = async (mode: RunMode) => { const performReIndexAll = async (data: CreateEventPublisherJob) => {
try { try {
setConfirmLoading(true);
await reIndexByPublisher({ await reIndexByPublisher({
runMode: mode, ...data,
entities, runMode: RunMode.Batch,
recreateIndex, } as CreateEventPublisherJob);
publisherType: PublisherType.ElasticSearch,
});
showSuccessToast(jsonData['api-success-messages']['fetch-re-index-all']); showSuccessToast(jsonData['api-success-messages']['fetch-re-index-all']);
} catch (err) { } catch (err) {
@ -112,6 +93,9 @@ const ElasticSearchIndexPage = () => {
err as AxiosError, err as AxiosError,
jsonData['api-error-messages']['update-re-index-all'] jsonData['api-error-messages']['update-re-index-all']
); );
} finally {
setModalOpen(false);
setConfirmLoading(false);
} }
}; };
@ -120,6 +104,25 @@ const ElasticSearchIndexPage = () => {
fetchStreamReIndexedData(); fetchStreamReIndexedData();
}; };
useEffect(() => {
if (socket) {
socket.on(SOCKET_EVENTS.JOB_STATUS, (newActivity) => {
if (newActivity) {
const activity = JSON.parse(newActivity) as EventPublisherJob;
if (activity.runMode === RunMode.Batch) {
setBatchJobData(activity);
} else {
setStreamJobData(activity);
}
}
});
}
return () => {
socket && socket.off(SOCKET_EVENTS.JOB_STATUS);
};
}, [socket]);
useEffect(() => { useEffect(() => {
fetchData(); fetchData();
}, []); }, []);
@ -128,277 +131,216 @@ const ElasticSearchIndexPage = () => {
<div> <div>
<Row gutter={[16, 16]}> <Row gutter={[16, 16]}>
<Col span={24}> <Col span={24}>
<Row gutter={[16, 16]}> <Card
<Col span={24}> extra={
<Card size="small"> <Space>
<div className="d-flex justify-between"> <Button
<div> data-testid="elastic-search-re-fetch-data"
<Skeleton loading={batchLoading}> disabled={streamLoading}
<Typography.Title level={5}> icon={<ReloadOutlined />}
ElasticSearch size="small"
</Typography.Title> title="Refresh log"
<Space direction="horizontal" size={16}> onClick={fetchBatchReIndexedData}
<div className="tw-flex"> />
<span className="tw-text-grey-muted">Mode</span> : <Button
<span className="tw-ml-2"> data-testid="elastic-search-re-index-all"
{startCase(batchJobData?.runMode) || '--'} disabled={!isAdminUser}
</span> size="small"
</div> type="primary"
<div className="tw-flex"> onClick={() => setModalOpen(true)}>
<span className="tw-text-grey-muted">Status</span> : Re Index All
<span className="tw-ml-2"> </Button>
<Space size={8}> </Space>
{batchJobData?.status && ( }
<SVGIcons loading={batchLoading}
alt="result" size="small"
className="w-4" title="ElasticSearch">
icon={getStatusResultBadgeIcon( <Row gutter={[16, 8]}>
batchJobData?.status <Col span={24}>
)} <Space wrap direction="horizontal" size={0}>
/> <div className="tw-flex">
)} <span className="tw-text-grey-muted">Mode</span> :
<span> <span className="tw-ml-2">
{getEventPublisherStatusText( {startCase(batchJobData?.runMode) || '--'}
batchJobData?.status </span>
) || '--'}
</span>
</Space>
</span>
</div>
<div className="tw-flex">
<span className="tw-text-grey-muted">
Index stats
</span>{' '}
:
<span className="tw-ml-2">
{!isEmpty(batchJobData) ? (
<Space size={8}>
<Badge
className="request-badge running"
count={batchJobData?.stats?.total}
overflowCount={99999999}
title={`Total index sent: ${batchJobData?.stats?.total}`}
/>
<Badge
className="request-badge success"
count={batchJobData?.stats?.success}
overflowCount={99999999}
title={`Success index: ${batchJobData?.stats?.success}`}
/>
<Badge
showZero
className="request-badge failed"
count={batchJobData?.stats?.failed}
overflowCount={99999999}
title={`Failed index: ${batchJobData?.stats?.failed}`}
/>
</Space>
) : (
'--'
)}
</span>
</div>
<div className="tw-flex">
<span className="tw-text-grey-muted">
Last Updated
</span>{' '}
:
<span className="tw-ml-2">
{batchJobData?.timestamp
? getDateTimeByTimeStampWithZone(
batchJobData?.timestamp
)
: '--'}
</span>
</div>
</Space>
<Space className="m-t-sm" size={16}>
<div>
<span className="tw-text-grey-muted">
Last Failed At:
</span>
<p className="tw-ml-2">
{batchJobData?.failureDetails?.lastFailedAt
? getDateTimeByTimeStampWithZone(
batchJobData?.failureDetails?.lastFailedAt
)
: '--'}
</p>
</div>
<div>
<span className="tw-text-grey-muted">
Last error:
</span>
<span className="tw-ml-2">
{batchJobData?.failureDetails?.lastFailedReason ? (
<RichTextEditorPreviewer
enableSeeMoreVariant={Boolean(batchJobData)}
markdown={
batchJobData?.failureDetails?.lastFailedReason
}
/>
) : (
'--'
)}
</span>
</div>
</Space>
</Skeleton>
</div> </div>
<Divider type="vertical" />
<Space <div className="tw-flex">
direction="vertical" <span className="tw-text-grey-muted">Status</span> :
size={16} <span className="tw-ml-2">
style={{ maxWidth: '420px' }}> <Space size={8}>
<Space size={8}> {batchJobData?.status && (
<Switch <SVGIcons
checked={recreateIndex} alt="result"
onChange={setRecreateIndex} className="w-4"
/> icon={getStatusResultBadgeIcon(
<Typography.Text batchJobData?.status
className="d-flex items-center" )}
type="secondary"> />
Recreate indexes&nbsp; )}
<Tooltip <span>
placement="bottomRight" {getEventPublisherStatusText(batchJobData?.status) ||
title="This will delete existing indexes and re-create them."> '--'}
<QuestionCircleOutlined /> </span>
</Tooltip> </Space>
</Typography.Text> </span>
</Space> </div>
<div> <Divider type="vertical" />
<Typography.Text className="m-b-sm"> <div className="tw-flex">
Entities <span className="tw-text-grey-muted">Index stats</span> :
</Typography.Text> <span className="tw-ml-2">
<Checkbox.Group {!isEmpty(batchJobData) ? (
defaultValue={entities}
onChange={(values) => setEntities(values as string[])}>
<Row>
{ELASTIC_SEARCH_INDEX_ENTITIES.map((option) => (
<Col key={option.value} span={8}>
<Checkbox value={option.value}>
{option.label}
</Checkbox>
</Col>
))}
</Row>
</Checkbox.Group>
</div>
<Space align="center" className="flex-end" size={16}>
<Button
data-testid="elastic-search-re-fetch-data"
disabled={batchLoading}
icon={<ReloadOutlined />}
onClick={fetchBatchReIndexedData}
/>
<Button
data-testid="elastic-search-re-index-all"
disabled={!isAdminUser}
type="primary"
onClick={() => performReIndexAll(RunMode.Batch)}>
Re Index All
</Button>
</Space>
</Space>
</div>
</Card>
</Col>
<Col span={24}>
<Card size="small">
<div className="d-flex justify-between">
<Typography.Title level={5}>ElasticSearch</Typography.Title>
<Space align="center" size={16}>
<Button
data-testid="elastic-search-re-fetch-data"
disabled={streamLoading}
icon={<ReloadOutlined />}
onClick={fetchStreamReIndexedData}
/>
</Space>
</div>
<Skeleton loading={streamLoading}>
<Space direction="horizontal" size={16}>
<div className="tw-flex">
<span className="tw-text-grey-muted">Mode</span> :
<span className="tw-ml-2">
{startCase(streamJobData?.runMode) || '--'}
</span>
</div>
<div className="tw-flex">
<span className="tw-text-grey-muted">Status</span> :
<span className="tw-ml-2">
<Space size={8}> <Space size={8}>
{streamJobData?.status && ( <Badge
<SVGIcons className="request-badge running"
alt="result" count={batchJobData?.stats?.total}
className="w-4" overflowCount={99999999}
icon={getStatusResultBadgeIcon( title={`Total index sent: ${batchJobData?.stats?.total}`}
streamJobData?.status />
)}
/>
)}
<span>
{getEventPublisherStatusText(
streamJobData?.status
) || '--'}
</span>
</Space>
</span>
</div>
<div className="tw-flex"> <Badge
<span className="tw-text-grey-muted">Last Updated</span> : className="request-badge success"
<span className="tw-ml-2"> count={batchJobData?.stats?.success}
{streamJobData?.timestamp overflowCount={99999999}
? getDateTimeByTimeStampWithZone( title={`Success index: ${batchJobData?.stats?.success}`}
streamJobData?.timestamp />
)
: '--'} <Badge
</span> showZero
</div> className="request-badge failed"
</Space> count={batchJobData?.stats?.failed}
<div> overflowCount={99999999}
<Space className="m-t-sm" size={16}> title={`Failed index: ${batchJobData?.stats?.failed}`}
<div> />
<span className="tw-text-grey-muted"> </Space>
Last Failed At: ) : (
</span> '--'
<p className="tw-ml-2"> )}
{streamJobData?.failureDetails?.lastFailedAt </span>
? getDateTimeByTimeStampWithZone(
streamJobData?.failureDetails?.lastFailedAt
)
: '--'}
</p>
</div>
<div>
<span className="tw-text-grey-muted">Last error</span> :
<span className="tw-ml-2">
{streamJobData?.failureDetails?.lastFailedReason ? (
<RichTextEditorPreviewer
enableSeeMoreVariant={Boolean(streamJobData)}
markdown={
streamJobData?.failureDetails?.lastFailedReason
}
/>
) : (
'--'
)}
</span>
</div>
</Space>
</div> </div>
</Skeleton> <Divider type="vertical" />
</Card> <div className="tw-flex">
</Col> <span className="tw-text-grey-muted">Last Updated</span> :
</Row> <span className="tw-ml-2">
{batchJobData?.timestamp
? getDateTimeByTimeStampWithZone(
batchJobData?.timestamp
)
: '--'}
</span>
</div>
<Divider type="vertical" />
<div className="tw-flex">
<span className="tw-text-grey-muted">Last Failed At:</span>
<p className="tw-ml-2">
{batchJobData?.failureDetails?.lastFailedAt
? getDateTimeByTimeStampWithZone(
batchJobData?.failureDetails?.lastFailedAt
)
: '--'}
</p>
</div>
</Space>
</Col>
<Col span={24}>
<span className="tw-text-grey-muted">Last error:</span>
<span className="tw-ml-2">
{batchJobData?.failureDetails?.lastFailedReason ? (
<RichTextEditorPreviewer
enableSeeMoreVariant={Boolean(batchJobData)}
markdown={batchJobData?.failureDetails?.lastFailedReason}
/>
) : (
'--'
)}
</span>
</Col>
</Row>
</Card>
</Col>
<Col span={24}>
<Card
extra={
<Button
data-testid="elastic-search-re-fetch-data"
disabled={streamLoading}
icon={<ReloadOutlined />}
size="small"
title="Refresh log"
onClick={fetchStreamReIndexedData}
/>
}
loading={streamLoading}
size="small"
title="ElasticSearch">
<Space direction="horizontal" size={16}>
<div className="tw-flex">
<span className="tw-text-grey-muted">Mode</span> :
<span className="tw-ml-2">
{startCase(streamJobData?.runMode) || '--'}
</span>
</div>
<div className="tw-flex">
<span className="tw-text-grey-muted">Status</span> :
<span className="tw-ml-2">
<Space size={8}>
{streamJobData?.status && (
<SVGIcons
alt="result"
className="w-4"
icon={getStatusResultBadgeIcon(streamJobData?.status)}
/>
)}
<span>
{getEventPublisherStatusText(streamJobData?.status) ||
'--'}
</span>
</Space>
</span>
</div>
<div className="tw-flex">
<span className="tw-text-grey-muted">Last Updated</span> :
<span className="tw-ml-2">
{streamJobData?.timestamp
? getDateTimeByTimeStampWithZone(streamJobData?.timestamp)
: '--'}
</span>
</div>
<div className="tw-flex">
<span className="tw-text-grey-muted">Last Failed At:</span>
<p className="tw-ml-2">
{streamJobData?.failureDetails?.lastFailedAt
? getDateTimeByTimeStampWithZone(
streamJobData?.failureDetails?.lastFailedAt
)
: '--'}
</p>
</div>
</Space>
<div>
<span className="tw-text-grey-muted">Last error</span> :
<span className="tw-ml-2">
{streamJobData?.failureDetails?.lastFailedReason ? (
<RichTextEditorPreviewer
enableSeeMoreVariant={Boolean(streamJobData)}
markdown={streamJobData?.failureDetails?.lastFailedReason}
/>
) : (
'--'
)}
</span>
</div>
</Card>
</Col> </Col>
</Row> </Row>
<ReIndexAllModal
confirmLoading={confirmLoading}
visible={isModalOpen}
onCancel={() => setModalOpen(false)}
onSave={performReIndexAll}
/>
</div> </div>
); );
}; };

1
openmetadata-ui/src/main/resources/ui/src/styles/antd-master.less
View File

@ -18,3 +18,4 @@
@import url('./components/table.less'); @import url('./components/table.less');
@import url('./components/toggle-switch.less'); @import url('./components/toggle-switch.less');
@import url('./components/button.less'); @import url('./components/button.less');
@import url('./components/card.less');

4
openmetadata-ui/src/main/resources/ui/src/styles/components/card.less Normal file
View File

@ -0,0 +1,4 @@
.ant-card {
box-shadow: @card-shadow;
border: 1px #dde3ea solid;
}

8
openmetadata-ui/src/main/resources/ui/src/styles/x-master.css
View File

@ -1060,14 +1060,6 @@ code {
background-color: #dbd1f9; background-color: #dbd1f9;
} }
.ant-select-single:not(.ant-select-customize-input) .ant-select-selector {
padding: 0px 4px;
}
.ant-select-single .ant-select-selector .ant-select-selection-search {
left: 4px;
}
@media only screen and (max-width: 1440px) { @media only screen and (max-width: 1440px) {
#left-panel { #left-panel {
width: 284px; width: 284px;