Remove ES reindexing files from UI (#13506)

* Remove ES reindexing files from UI

* fix minor errors

---------

Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com>
This commit is contained in:
Mohit Yadav 2023-10-10 14:28:40 +05:30 committed by GitHub
parent f69cd9f54a
commit d3fd0237e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 21 additions and 559 deletions

View File

@ -1,198 +0,0 @@
/*
* Copyright 2023 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { AxiosError } from 'axios';
import { isEqual } from 'lodash';
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useParams } from 'react-router-dom';
import { useWebSocketConnector } from '../../components/web-scoket/web-scoket.provider';
import { SOCKET_EVENTS } from '../../constants/constants';
import {
ELASTIC_SEARCH_INDEX_ENTITIES,
ELASTIC_SEARCH_INITIAL_VALUES,
} from '../../constants/elasticsearch.constant';
import { ELASTIC_SEARCH_RE_INDEX_PAGE_TABS } from '../../enums/ElasticSearch.enum';
import { CreateEventPublisherJob } from '../../generated/api/createEventPublisherJob';
import {
EventPublisherJob,
RunMode,
} from '../../generated/system/eventPublisherJob';
import {
getBatchJobReIndexStatus,
getStreamJobReIndexStatus,
reIndexByPublisher,
stopBatchJobReIndex,
} from '../../rest/elasticSearchReIndexAPI';
import { getJobDetailsCard } from '../../utils/EventPublisherUtils';
import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils';
import ReIndexAllModal from './ElasticSearchReIndexModal.component';
function TriggerReIndexing() {
const { t } = useTranslation();
const { fqn } = useParams<{ fqn: string }>();
const [batchJobData, setBatchJobData] = useState<EventPublisherJob>();
const [streamJobData, setStreamJobData] = useState<EventPublisherJob>();
const [batchLoading, setBatchLoading] = useState(false);
const [streamLoading, setStreamLoading] = useState(false);
const [confirmLoading, setConfirmLoading] = useState(false);
const [isModalOpen, setIsModalOpen] = useState(false);
const { socket } = useWebSocketConnector();
const isOnDemandTab = useMemo(
() => fqn === ELASTIC_SEARCH_RE_INDEX_PAGE_TABS.ON_DEMAND,
[fqn]
);
const showReIndexAllModal = () => {
setIsModalOpen(true);
};
const fetchBatchReIndexedData = useCallback(async () => {
try {
setBatchLoading(true);
const response = await getBatchJobReIndexStatus();
setBatchJobData(response);
} catch (error) {
showErrorToast(
error as AxiosError,
t('server.fetch-re-index-data-error')
);
} finally {
setBatchLoading(false);
}
}, [setBatchJobData, setBatchLoading]);
const stopBatchReIndexedJob = useCallback(async () => {
if (batchJobData) {
try {
await stopBatchJobReIndex(batchJobData?.id);
showSuccessToast(t('server.re-indexing-stopped'));
} catch (error) {
showErrorToast(error as AxiosError, t('server.stop-re-indexing-error'));
}
}
}, [batchJobData]);
const fetchStreamReIndexedData = useCallback(async () => {
try {
setStreamLoading(true);
const response = await getStreamJobReIndexStatus();
setStreamJobData(response);
} catch (error) {
// Error will be logged to console
} finally {
setStreamLoading(false);
}
}, [setStreamJobData, setStreamLoading]);
const performReIndexAll = useCallback(
async (data: CreateEventPublisherJob) => {
try {
setConfirmLoading(true);
await reIndexByPublisher({
...data,
entities: isEqual(
data.entities,
ELASTIC_SEARCH_INITIAL_VALUES.entities
)
? ELASTIC_SEARCH_INDEX_ENTITIES.map((e) => e.value)
: data.entities ?? [],
runMode: RunMode.Batch,
} as CreateEventPublisherJob);
showSuccessToast(t('server.re-indexing-started'));
} catch (err) {
showErrorToast(err as AxiosError, t('server.re-indexing-error'));
} finally {
setIsModalOpen(false);
setConfirmLoading(false);
}
},
[setIsModalOpen, setConfirmLoading]
);
useEffect(() => {
if (socket) {
socket.on(SOCKET_EVENTS.JOB_STATUS, (newActivity) => {
if (newActivity) {
const activity = JSON.parse(newActivity) as EventPublisherJob;
if (activity.runMode === RunMode.Batch) {
setBatchJobData(activity);
} else {
setStreamJobData(activity);
}
}
});
}
return () => {
socket && socket.off(SOCKET_EVENTS.JOB_STATUS);
};
}, [socket]);
const jobDetailsCard = useMemo(
() =>
isOnDemandTab
? getJobDetailsCard(
batchLoading,
fetchBatchReIndexedData,
batchJobData,
batchJobData?.failure?.sourceError,
showReIndexAllModal,
stopBatchReIndexedJob
)
: getJobDetailsCard(
streamLoading,
fetchStreamReIndexedData,
streamJobData,
streamJobData?.failure?.sinkError
),
[
isOnDemandTab,
batchLoading,
streamLoading,
fetchBatchReIndexedData,
fetchStreamReIndexedData,
batchJobData,
streamJobData,
]
);
useEffect(() => {
if (isOnDemandTab) {
fetchBatchReIndexedData();
} else {
fetchStreamReIndexedData();
}
}, [isOnDemandTab]);
return (
<>
{jobDetailsCard}
<ReIndexAllModal
confirmLoading={confirmLoading}
visible={isModalOpen}
onCancel={() => setIsModalOpen(false)}
onSave={performReIndexAll}
/>
</>
);
}
export default TriggerReIndexing;

View File

@ -105,15 +105,6 @@ const UserListPageV1 = withSuspenseFallback(
React.lazy(() => import('../../pages/UserListPage/UserListPageV1'))
);
const ElasticSearchIndexPage = withSuspenseFallback(
React.lazy(
() =>
import(
'../../pages/ElasticSearchIndexPage/ElasticSearchReIndexPage.component'
)
)
);
const DataInsightsSettingsPage = withSuspenseFallback(
React.lazy(
() =>
@ -244,17 +235,6 @@ const GlobalSettingRouter = () => {
)}
/>
<AdminProtectedRoute
exact
component={ElasticSearchIndexPage}
hasPermission={false}
path={getSettingPath(
GlobalSettingsMenuCategory.OPEN_METADATA,
GlobalSettingOptions.SEARCH,
true
)}
/>
<AdminProtectedRoute
exact
component={DataInsightsSettingsPage}
@ -359,6 +339,7 @@ const GlobalSettingRouter = () => {
true
)}
/>
<AdminProtectedRoute
exact
component={AlertDataInsightReportPage}

View File

@ -13,7 +13,10 @@
import i18next from 'i18next';
import { StepperStepType } from 'Models';
import { FilterPattern } from '../generated/entity/services/ingestionPipelines/ingestionPipeline';
import {
FilterPattern,
PipelineType,
} from '../generated/entity/services/ingestionPipelines/ingestionPipeline';
export const STEPS_FOR_ADD_INGESTION: Array<StepperStepType> = [
{
@ -36,14 +39,15 @@ export const INGESTION_ACTION_TYPE = {
};
export const PIPELINE_TYPE_LOCALIZATION = {
dataInsight: 'data-insight',
dbt: 'dbt-lowercase',
elasticSearchReindex: 'elastic-search-re-index',
lineage: 'lineage',
metadata: 'metadata',
profiler: 'profiler',
TestSuite: 'test-suite',
usage: 'usage',
[PipelineType.DataInsight]: 'data-insight',
[PipelineType.Dbt]: 'dbt-lowercase',
[PipelineType.ElasticSearchReindex]: 'elastic-search-re-index',
[PipelineType.Lineage]: 'lineage',
[PipelineType.Metadata]: 'metadata',
[PipelineType.Profiler]: 'profiler',
[PipelineType.TestSuite]: 'test-suite',
[PipelineType.Usage]: 'usage',
[PipelineType.Application]: 'application',
};
export const DBT_CLASSIFICATION_DEFAULT_VALUE = 'dbtTags';

View File

@ -1,88 +0,0 @@
/*
* Copyright 2022 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Col, Row, Tabs, TabsProps } from 'antd';
import React, { useCallback, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { useHistory, useParams } from 'react-router-dom';
import PageHeader from '../../components/header/PageHeader.component';
import SettingsIngestion from '../../components/SettingsIngestion/SettingsIngestion.component';
import TriggerReIndexing from '../../components/TriggerReIndexing/TriggerReIndexing.component';
import {
GlobalSettingOptions,
GlobalSettingsMenuCategory,
} from '../../constants/GlobalSettings.constants';
import { ELASTIC_SEARCH_RE_INDEX_PAGE_TABS } from '../../enums/ElasticSearch.enum';
import { PipelineType } from '../../generated/api/services/ingestionPipelines/createIngestionPipeline';
import { getSettingsPathWithFqn } from '../../utils/RouterUtils';
import './ElasticSearchReIndex.style.less';
const ElasticSearchIndexPage = () => {
const { t } = useTranslation();
const history = useHistory();
const { fqn } = useParams<{ fqn: string }>();
const tabItems: TabsProps['items'] = useMemo(
() => [
{
key: ELASTIC_SEARCH_RE_INDEX_PAGE_TABS.ON_DEMAND,
label: t('label.on-demand'),
children: <TriggerReIndexing />,
},
{
key: ELASTIC_SEARCH_RE_INDEX_PAGE_TABS.LIVE,
label: t('label.live'),
children: <TriggerReIndexing />,
},
{
key: ELASTIC_SEARCH_RE_INDEX_PAGE_TABS.SCHEDULE,
label: t('label.schedule'),
children: (
<SettingsIngestion
containerClassName="m-t-0"
pipelineType={PipelineType.ElasticSearchReindex}
/>
),
},
],
[]
);
const handleTabClick = useCallback((activeKey: string) => {
history.replace(
getSettingsPathWithFqn(
GlobalSettingsMenuCategory.OPEN_METADATA,
GlobalSettingOptions.SEARCH,
activeKey
)
);
}, []);
return (
<Row align="middle" gutter={[16, 16]}>
<Col span={24}>
<PageHeader
data={{
header: t('label.search'),
subHeader: t('message.elastic-search-message'),
}}
/>
</Col>
<Col span={24}>
<Tabs activeKey={fqn} items={tabItems} onTabClick={handleTabClick} />
</Col>
</Row>
);
};
export default ElasticSearchIndexPage;

View File

@ -13,14 +13,13 @@
import { AxiosResponse } from 'axios';
import axiosClient from '.';
import { CreateEventPublisherJob } from '../generated/api/createEventPublisherJob';
import {
EventPublisherJob,
CreateEventPublisherJob,
PublisherType,
} from '../generated/system/eventPublisherJob';
} from '../generated/api/createEventPublisherJob';
export const getStreamJobReIndexStatus = async () => {
const res = await axiosClient.get<EventPublisherJob>(
const res = await axiosClient.get<CreateEventPublisherJob>(
`/search/reindex/stream/status`
);
@ -28,7 +27,9 @@ export const getStreamJobReIndexStatus = async () => {
};
export const getBatchJobReIndexStatus = async () => {
const res = await axiosClient.get<EventPublisherJob>(`search/reindex/latest`);
const res = await axiosClient.get<CreateEventPublisherJob>(
`search/reindex/latest`
);
return res.data;
};
@ -47,7 +48,7 @@ export const reIndexByPublisher = async (data: CreateEventPublisherJob) => {
const res = await axiosClient.post<
CreateEventPublisherJob,
AxiosResponse<EventPublisherJob>
AxiosResponse<CreateEventPublisherJob>
>('/search/reindex', payload);
return res.data;

View File

@ -1,238 +0,0 @@
/*
* Copyright 2022 Collate.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ReloadOutlined } from '@ant-design/icons';
import { Badge, Button, Card, Col, Divider, Row, Space } from 'antd';
import { t } from 'i18next';
import { isEmpty, startCase } from 'lodash';
import React from 'react';
import { ReactComponent as IconFailBadge } from '../assets/svg/fail-badge.svg';
import { ReactComponent as IconTaskOpen } from '../assets/svg/in-progress.svg';
import { ReactComponent as IconTaskStopped } from '../assets/svg/pending-badge.svg';
import { ReactComponent as IconSuccessBadge } from '../assets/svg/success-badge.svg';
import RichTextEditorPreviewer from '../components/common/rich-text-editor/RichTextEditorPreviewer';
import Loader from '../components/Loader/Loader';
import {
EventPublisherJob,
SourceError,
Status,
} from '../generated/system/eventPublisherJob';
import { formatDateTimeWithTimezone } from './date-time/DateTimeUtils';
export const getStatusResultBadgeIcon = (status?: string) => {
switch (status) {
case Status.Stopped:
return <IconTaskStopped height={14} width={14} />;
case Status.Completed:
return <IconSuccessBadge height={14} width={14} />;
case Status.Failed:
case Status.ActiveWithError:
return <IconFailBadge height={14} width={14} />;
case Status.Running:
case Status.Started:
return <Loader size="x-small" />;
case Status.Active:
default:
return <IconTaskOpen height={14} width={14} />;
}
};
export const getEventPublisherStatusText = (status?: string) => {
switch (status) {
case Status.Stopped:
return t('label.stopped');
case Status.Failed:
return t('label.failed');
case Status.Running:
return t('label.running');
case Status.Completed:
return t('label.completed');
case Status.Active:
return t('label.active');
case Status.ActiveWithError:
return t('label.active-with-error');
case Status.Started:
return t('label.started');
default:
return status || '';
}
};
export const getJobDetailsCard = (
loadingState: boolean,
fetchJobData: () => Promise<void>,
jobData?: EventPublisherJob,
error?: SourceError,
showReIndexAllModal?: () => void,
stopBatchReIndexedJob?: () => void
) => {
return (
<Card
extra={
<Space>
<Button
data-testid="elastic-search-re-fetch-data"
disabled={loadingState}
icon={<ReloadOutlined />}
size="small"
title={t('label.refresh-log')}
onClick={fetchJobData}
/>
{showReIndexAllModal &&
(jobData?.status === Status.Running ? (
<Button
data-testid="elastic-search-stop-batch-re-index"
size="small"
type="primary"
onClick={stopBatchReIndexedJob}>
{t('label.stop-re-index-all')}
</Button>
) : (
<Button
data-testid="elastic-search-re-index-all"
size="small"
type="primary"
onClick={showReIndexAllModal}>
{t('label.re-index-all')}
</Button>
))}
</Space>
}
loading={loadingState}
size="small"
title={t('label.elasticsearch')}>
<Row gutter={[16, 8]}>
<Col span={24}>
<Space wrap direction="horizontal" size={0}>
<div className="flex">
<span className="text-grey-muted">{`${t('label.mode')}:`}</span>
<span className="m-l-xs">
{startCase(jobData?.runMode) || '--'}
</span>
</div>
<Divider type="vertical" />
<div className="flex">
<span className="text-grey-muted">{`${t('label.status')}:`}</span>
<Space align="center" className="m-l-xs" size={8}>
{getStatusResultBadgeIcon(jobData?.status)}
<span>
{getEventPublisherStatusText(jobData?.status) || '--'}
</span>
</Space>
</div>
<Divider type="vertical" />
{showReIndexAllModal && (
<div className="flex">
<span className="text-grey-muted">{`${t(
'label.index-states'
)}:`}</span>
<span className="m-l-xs">
{!isEmpty(jobData) ? (
<Space size={8}>
<Badge
className="request-badge running"
count={jobData?.stats?.jobStats?.totalRecords}
overflowCount={99999999}
title={`${t('label.total-index-sent')}: ${
jobData?.stats?.jobStats?.totalRecords
}`}
/>
<Badge
className="request-badge success"
count={jobData?.stats?.jobStats?.successRecords}
overflowCount={99999999}
title={`${t('label.entity-index', {
entity: t('label.success'),
})}: ${jobData?.stats?.jobStats?.successRecords}`}
/>
<Badge
showZero
className="request-badge failed"
count={jobData?.stats?.jobStats?.failedRecords}
overflowCount={99999999}
title={`${t('label.entity-index', {
entity: t('label.failed'),
})}: ${jobData?.stats?.jobStats?.failedRecords}`}
/>
</Space>
) : (
'--'
)}
</span>
</div>
)}
<Divider type="vertical" />
<div className="flex">
<span className="text-grey-muted">{`${t(
'label.last-updated'
)}:`}</span>
<span className="m-l-xs">
{jobData?.timestamp
? formatDateTimeWithTimezone(jobData?.timestamp)
: '--'}
</span>
</div>
<Divider type="vertical" />
<div className="flex">
<span className="text-grey-muted">{`${t(
'label.last-failed-at'
)}:`}</span>
<p className="m-l-xs">
{error
? formatDateTimeWithTimezone(error?.lastFailedAt ?? 0)
: '--'}
</p>
</div>
</Space>
</Col>
<Col span={24}>
<span className="text-grey-muted">{`${t(
'label.failure-context'
)}:`}</span>
<span className="m-l-xs">
{error?.context ? (
<RichTextEditorPreviewer
enableSeeMoreVariant={Boolean(jobData)}
markdown={error?.context}
/>
) : (
'--'
)}
</span>
</Col>
<Col span={24}>
<span className="text-grey-muted">{`${t('label.last-error')}:`}</span>
<span className="m-l-xs">
{error?.lastFailedReason ? (
<RichTextEditorPreviewer
enableSeeMoreVariant={Boolean(jobData)}
markdown={error?.lastFailedReason}
/>
) : (
'--'
)}
</span>
</Col>
</Row>
</Card>
);
};