fix(ui): support batchSize & flushIntervalInSec while ES re-indexing (#8082)

* fix(ui): support batchSize & flushIntervalInSec while ES re-indexing

* add refresh button for stream mode

* add refresh batch for fallback option

* update flush interval label

openmetadata-ui/src/main/resources/ui/src/axiosAPIs/elastic-index-API.ts

@@ -31,12 +31,16 @@ export const reIndexByPublisher = async ({
   runMode,
   entities = ['all'],
   recreateIndex = true,
+  batchSize,
+  flushIntervalInSec,
 }: CreateEventPublisherJob) => {
   const payload = {
     publisherType: PublisherType.ElasticSearch,
     runMode,
     recreateIndex,
     entities,
+    batchSize,
+    flushIntervalInSec,
   };
 
   const res = await axiosClient.post('/indexResource/reindex', payload);

openmetadata-ui/src/main/resources/ui/src/constants/constants.ts

@@ -258,6 +258,7 @@ export const SOCKET_EVENTS = {
   ACTIVITY_FEED: 'activityFeed',
   TASK_CHANNEL: 'taskChannel',
   MENTION_CHANNEL: 'mentionChannel',
+  JOB_STATUS: 'jobStatus',
 };
 
 export const IN_PAGE_SEARCH_ROUTES: Record<string, Array<string>> = {

openmetadata-ui/src/main/resources/ui/src/constants/elasticsearch.constant.ts

@@ -49,3 +49,32 @@ export const ELASTIC_SEARCH_INDEX_ENTITIES = [
     label: 'Tag',
   },
 ];
+
+export const ELASTIC_SEARCH_INITIAL_VALUES = {
+  entities: [
+    'table',
+    'topic',
+    'dashboard',
+    'pipeline',
+    'mlmodel',
+    'bot',
+    'user',
+    'team',
+    'glossaryTerm',
+    'tag',
+  ],
+  batchSize: 100,
+  flushIntervalInSec: 30,
+  recreateIndex: false,
+};
+
+export const RECREATE_INDEX_OPTIONS = [
+  {
+    label: 'Yes',
+    value: true,
+  },
+  {
+    label: 'No',
+    value: false,
+  },
+];

openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/elastic-re-index-modal.component.tsx

@@ -0,0 +1,102 @@
+/*
+ *  Copyright 2022 Collate
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+import { Checkbox, Col, Form, Input, Modal, Row, Select } from 'antd';
+import React, { useState } from 'react';
+import {
+  ELASTIC_SEARCH_INDEX_ENTITIES,
+  ELASTIC_SEARCH_INITIAL_VALUES,
+  RECREATE_INDEX_OPTIONS,
+} from '../../constants/elasticsearch.constant';
+import { CreateEventPublisherJob } from '../../generated/api/createEventPublisherJob';
+
+interface ReIndexAllModalInterface {
+  visible: boolean;
+  onCancel: () => void;
+  onSave?: (data: CreateEventPublisherJob) => void;
+  confirmLoading: boolean;
+}
+
+const ReIndexAllModal = ({
+  visible,
+  onCancel,
+  onSave,
+  confirmLoading,
+}: ReIndexAllModalInterface) => {
+  const [entities, setEntities] = useState<string[]>(
+    ELASTIC_SEARCH_INITIAL_VALUES.entities
+  );
+
+  return (
+    <Modal
+      centered
+      confirmLoading={confirmLoading}
+      okButtonProps={{
+        form: 're-index-form',
+        type: 'primary',
+        htmlType: 'submit',
+      }}
+      okText="Submit"
+      title="Re-Index Elastic Search"
+      visible={visible}
+      width={650}
+      onCancel={onCancel}>
+      <Form
+        id="re-index-form"
+        layout="vertical"
+        name="elastic-search-re-index"
+        onFinish={onSave}>
+        <Form.Item
+          initialValue={false}
+          label="Recreate indexes"
+          name="recreateIndex">
+          <Select
+            data-testid="re-index-selector"
+            options={RECREATE_INDEX_OPTIONS}
+          />
+        </Form.Item>
+
+        <Form.Item initialValue={entities} label="Entities" name="entities">
+          <Checkbox.Group
+            onChange={(values) => setEntities(values as string[])}>
+            <Row gutter={[16, 16]}>
+              {ELASTIC_SEARCH_INDEX_ENTITIES.map((option) => (
+                <Col key={option.value} span={6}>
+                  <Checkbox value={option.value}>{option.label}</Checkbox>
+                </Col>
+              ))}
+            </Row>
+          </Checkbox.Group>
+        </Form.Item>
+        <Form.Item
+          initialValue={ELASTIC_SEARCH_INITIAL_VALUES.flushIntervalInSec}
+          label="Flush Interval (secs):"
+          name="flushIntervalInSec">
+          <Input
+            data-testid="flush-interval-in-sec"
+            placeholder="Enter seconds"
+          />
+        </Form.Item>
+
+        <Form.Item
+          initialValue={ELASTIC_SEARCH_INITIAL_VALUES.batchSize}
+          label="Batch Size:"
+          name="batchSize">
+          <Input data-testid="batch-size" placeholder="Enter batch size" />
+        </Form.Item>
+      </Form>
+    </Modal>
+  );
+};
+
+export default ReIndexAllModal;

openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/elastic-search-index.page.tsx

@@ -11,20 +11,8 @@
  *  limitations under the License.
  */
 
-import { QuestionCircleOutlined, ReloadOutlined } from '@ant-design/icons';
+import { ReloadOutlined } from '@ant-design/icons';
-import {
+import { Badge, Button, Card, Col, Divider, Row, Space } from 'antd';
-  Badge,
-  Button,
-  Card,
-  Checkbox,
-  Col,
-  Row,
-  Skeleton,
-  Space,
-  Switch,
-  Tooltip,
-  Typography,
-} from 'antd';
 import { AxiosError } from 'axios';
 import { isEmpty, startCase } from 'lodash';
 import React, { useEffect, useState } from 'react';
@@ -33,10 +21,11 @@ import {
   reIndexByPublisher,
 } from '../../axiosAPIs/elastic-index-API';
 import RichTextEditorPreviewer from '../../components/common/rich-text-editor/RichTextEditorPreviewer';
-import { ELASTIC_SEARCH_INDEX_ENTITIES } from '../../constants/elasticsearch.constant';
+import { useWebSocketConnector } from '../../components/web-scoket/web-scoket.provider';
+import { SOCKET_EVENTS } from '../../constants/constants';
+import { CreateEventPublisherJob } from '../../generated/api/createEventPublisherJob';
 import {
   EventPublisherJob,
-  PublisherType,
   RunMode,
 } from '../../generated/settings/eventPublisherJob';
 import { useAuth } from '../../hooks/authHooks';
@@ -48,6 +37,7 @@ import {
 import SVGIcons from '../../utils/SvgUtils';
 import { getDateTimeByTimeStampWithZone } from '../../utils/TimeUtils';
 import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils';
+import ReIndexAllModal from './elastic-re-index-modal.component';
 import './elastic-search-index.style.less';
 
 const ElasticSearchIndexPage = () => {
@@ -57,19 +47,11 @@ const ElasticSearchIndexPage = () => {
   const { isAdminUser } = useAuth();
   const [batchLoading, setBatchLoading] = useState(false);
   const [streamLoading, setStreamLoading] = useState(false);
-  const [recreateIndex, setRecreateIndex] = useState(false);
+  const [confirmLoading, setConfirmLoading] = useState(false);
-  const [entities, setEntities] = useState<string[]>([
+
-    'table',
+  const [isModalOpen, setModalOpen] = useState(false);
-    'topic',
+
-    'dashboard',
+  const { socket } = useWebSocketConnector();
-    'pipeline',
-    'mlmodel',
-    'bot',
-    'user',
-    'team',
-    'glossaryTerm',
-    'tag',
-  ]);
 
   const fetchBatchReIndexedData = async () => {
     try {
@@ -97,14 +79,13 @@ const ElasticSearchIndexPage = () => {
     }
   };
 
-  const performReIndexAll = async (mode: RunMode) => {
+  const performReIndexAll = async (data: CreateEventPublisherJob) => {
     try {
+      setConfirmLoading(true);
       await reIndexByPublisher({
-        runMode: mode,
+        ...data,
-        entities,
+        runMode: RunMode.Batch,
-        recreateIndex,
+      } as CreateEventPublisherJob);
-        publisherType: PublisherType.ElasticSearch,
-      });
 
       showSuccessToast(jsonData['api-success-messages']['fetch-re-index-all']);
     } catch (err) {
@@ -112,6 +93,9 @@ const ElasticSearchIndexPage = () => {
         err as AxiosError,
         jsonData['api-error-messages']['update-re-index-all']
       );
+    } finally {
+      setModalOpen(false);
+      setConfirmLoading(false);
     }
   };
 
@@ -120,6 +104,25 @@ const ElasticSearchIndexPage = () => {
     fetchStreamReIndexedData();
   };
 
+  useEffect(() => {
+    if (socket) {
+      socket.on(SOCKET_EVENTS.JOB_STATUS, (newActivity) => {
+        if (newActivity) {
+          const activity = JSON.parse(newActivity) as EventPublisherJob;
+          if (activity.runMode === RunMode.Batch) {
+            setBatchJobData(activity);
+          } else {
+            setStreamJobData(activity);
+          }
+        }
+      });
+    }
+
+    return () => {
+      socket && socket.off(SOCKET_EVENTS.JOB_STATUS);
+    };
+  }, [socket]);
+
   useEffect(() => {
     fetchData();
   }, []);
@@ -128,22 +131,40 @@ const ElasticSearchIndexPage = () => {
     <div>
       <Row gutter={[16, 16]}>
         <Col span={24}>
-          <Row gutter={[16, 16]}>
+          <Card
+            extra={
+              <Space>
+                <Button
+                  data-testid="elastic-search-re-fetch-data"
+                  disabled={streamLoading}
+                  icon={<ReloadOutlined />}
+                  size="small"
+                  title="Refresh log"
+                  onClick={fetchBatchReIndexedData}
+                />
+                <Button
+                  data-testid="elastic-search-re-index-all"
+                  disabled={!isAdminUser}
+                  size="small"
+                  type="primary"
+                  onClick={() => setModalOpen(true)}>
+                  Re Index All
+                </Button>
+              </Space>
+            }
+            loading={batchLoading}
+            size="small"
+            title="ElasticSearch">
+            <Row gutter={[16, 8]}>
               <Col span={24}>
-              <Card size="small">
+                <Space wrap direction="horizontal" size={0}>
-                <div className="d-flex justify-between">
-                  <div>
-                    <Skeleton loading={batchLoading}>
-                      <Typography.Title level={5}>
-                        ElasticSearch
-                      </Typography.Title>
-                      <Space direction="horizontal" size={16}>
                   <div className="tw-flex">
                     <span className="tw-text-grey-muted">Mode</span> :
                     <span className="tw-ml-2">
                       {startCase(batchJobData?.runMode) || '--'}
                     </span>
                   </div>
+                  <Divider type="vertical" />
                   <div className="tw-flex">
                     <span className="tw-text-grey-muted">Status</span> :
                     <span className="tw-ml-2">
@@ -158,19 +179,15 @@ const ElasticSearchIndexPage = () => {
                           />
                         )}
                         <span>
-                                {getEventPublisherStatusText(
+                          {getEventPublisherStatusText(batchJobData?.status) ||
-                                  batchJobData?.status
+                            '--'}
-                                ) || '--'}
                         </span>
                       </Space>
                     </span>
                   </div>
-
+                  <Divider type="vertical" />
                   <div className="tw-flex">
-                          <span className="tw-text-grey-muted">
+                    <span className="tw-text-grey-muted">Index stats</span> :
-                            Index stats
-                          </span>{' '}
-                          :
                     <span className="tw-ml-2">
                       {!isEmpty(batchJobData) ? (
                         <Space size={8}>
@@ -201,12 +218,9 @@ const ElasticSearchIndexPage = () => {
                       )}
                     </span>
                   </div>
-
+                  <Divider type="vertical" />
                   <div className="tw-flex">
-                          <span className="tw-text-grey-muted">
+                    <span className="tw-text-grey-muted">Last Updated</span> :
-                            Last Updated
-                          </span>{' '}
-                          :
                     <span className="tw-ml-2">
                       {batchJobData?.timestamp
                         ? getDateTimeByTimeStampWithZone(
@@ -215,12 +229,9 @@ const ElasticSearchIndexPage = () => {
                         : '--'}
                     </span>
                   </div>
-                      </Space>
+                  <Divider type="vertical" />
-                      <Space className="m-t-sm" size={16}>
+                  <div className="tw-flex">
-                        <div>
+                    <span className="tw-text-grey-muted">Last Failed At:</span>
-                          <span className="tw-text-grey-muted">
-                            Last Failed At:
-                          </span>
                     <p className="tw-ml-2">
                       {batchJobData?.failureDetails?.lastFailedAt
                         ? getDateTimeByTimeStampWithZone(
@@ -229,99 +240,40 @@ const ElasticSearchIndexPage = () => {
                         : '--'}
                     </p>
                   </div>
-                        <div>
+                </Space>
-                          <span className="tw-text-grey-muted">
+              </Col>
-                            Last error:
+
-                          </span>
+              <Col span={24}>
+                <span className="tw-text-grey-muted">Last error:</span>
                 <span className="tw-ml-2">
                   {batchJobData?.failureDetails?.lastFailedReason ? (
                     <RichTextEditorPreviewer
                       enableSeeMoreVariant={Boolean(batchJobData)}
-                                markdown={
+                      markdown={batchJobData?.failureDetails?.lastFailedReason}
-                                  batchJobData?.failureDetails?.lastFailedReason
-                                }
                     />
                   ) : (
                     '--'
                   )}
                 </span>
-                        </div>
-                      </Space>
-                    </Skeleton>
-                  </div>
-
-                  <Space
-                    direction="vertical"
-                    size={16}
-                    style={{ maxWidth: '420px' }}>
-                    <Space size={8}>
-                      <Switch
-                        checked={recreateIndex}
-                        onChange={setRecreateIndex}
-                      />
-                      <Typography.Text
-                        className="d-flex items-center"
-                        type="secondary">
-                        Recreate indexes&nbsp;
-                        <Tooltip
-                          placement="bottomRight"
-                          title="This will delete existing indexes and re-create them.">
-                          <QuestionCircleOutlined />
-                        </Tooltip>
-                      </Typography.Text>
-                    </Space>
-                    <div>
-                      <Typography.Text className="m-b-sm">
-                        Entities
-                      </Typography.Text>
-                      <Checkbox.Group
-                        defaultValue={entities}
-                        onChange={(values) => setEntities(values as string[])}>
-                        <Row>
-                          {ELASTIC_SEARCH_INDEX_ENTITIES.map((option) => (
-                            <Col key={option.value} span={8}>
-                              <Checkbox value={option.value}>
-                                {option.label}
-                              </Checkbox>
               </Col>
-                          ))}
             </Row>
-                      </Checkbox.Group>
-                    </div>
-
-                    <Space align="center" className="flex-end" size={16}>
-                      <Button
-                        data-testid="elastic-search-re-fetch-data"
-                        disabled={batchLoading}
-                        icon={<ReloadOutlined />}
-                        onClick={fetchBatchReIndexedData}
-                      />
-                      <Button
-                        data-testid="elastic-search-re-index-all"
-                        disabled={!isAdminUser}
-                        type="primary"
-                        onClick={() => performReIndexAll(RunMode.Batch)}>
-                        Re Index All
-                      </Button>
-                    </Space>
-                  </Space>
-                </div>
           </Card>
         </Col>
         <Col span={24}>
-              <Card size="small">
+          <Card
-                <div className="d-flex justify-between">
+            extra={
-                  <Typography.Title level={5}>ElasticSearch</Typography.Title>
-                  <Space align="center" size={16}>
               <Button
                 data-testid="elastic-search-re-fetch-data"
                 disabled={streamLoading}
                 icon={<ReloadOutlined />}
+                size="small"
+                title="Refresh log"
                 onClick={fetchStreamReIndexedData}
               />
-                  </Space>
+            }
-                </div>
+            loading={streamLoading}
-                <Skeleton loading={streamLoading}>
+            size="small"
+            title="ElasticSearch">
             <Space direction="horizontal" size={16}>
               <div className="tw-flex">
                 <span className="tw-text-grey-muted">Mode</span> :
@@ -337,15 +289,12 @@ const ElasticSearchIndexPage = () => {
                       <SVGIcons
                         alt="result"
                         className="w-4"
-                              icon={getStatusResultBadgeIcon(
+                        icon={getStatusResultBadgeIcon(streamJobData?.status)}
-                                streamJobData?.status
-                              )}
                       />
                     )}
                     <span>
-                            {getEventPublisherStatusText(
+                      {getEventPublisherStatusText(streamJobData?.status) ||
-                              streamJobData?.status
+                        '--'}
-                            ) || '--'}
                     </span>
                   </Space>
                 </span>
@@ -355,19 +304,12 @@ const ElasticSearchIndexPage = () => {
                 <span className="tw-text-grey-muted">Last Updated</span> :
                 <span className="tw-ml-2">
                   {streamJobData?.timestamp
-                          ? getDateTimeByTimeStampWithZone(
+                    ? getDateTimeByTimeStampWithZone(streamJobData?.timestamp)
-                              streamJobData?.timestamp
-                            )
                     : '--'}
                 </span>
               </div>
-                  </Space>
+              <div className="tw-flex">
-                  <div>
+                <span className="tw-text-grey-muted">Last Failed At:</span>
-                    <Space className="m-t-sm" size={16}>
-                      <div>
-                        <span className="tw-text-grey-muted">
-                          Last Failed At:
-                        </span>
                 <p className="tw-ml-2">
                   {streamJobData?.failureDetails?.lastFailedAt
                     ? getDateTimeByTimeStampWithZone(
@@ -376,29 +318,29 @@ const ElasticSearchIndexPage = () => {
                     : '--'}
                 </p>
               </div>
+            </Space>
             <div>
               <span className="tw-text-grey-muted">Last error</span> :
               <span className="tw-ml-2">
                 {streamJobData?.failureDetails?.lastFailedReason ? (
                   <RichTextEditorPreviewer
                     enableSeeMoreVariant={Boolean(streamJobData)}
-                              markdown={
+                    markdown={streamJobData?.failureDetails?.lastFailedReason}
-                                streamJobData?.failureDetails?.lastFailedReason
-                              }
                   />
                 ) : (
                   '--'
                 )}
               </span>
             </div>
-                    </Space>
-                  </div>
-                </Skeleton>
           </Card>
         </Col>
       </Row>
-        </Col>
+      <ReIndexAllModal
-      </Row>
+        confirmLoading={confirmLoading}
+        visible={isModalOpen}
+        onCancel={() => setModalOpen(false)}
+        onSave={performReIndexAll}
+      />
     </div>
   );
 };

openmetadata-ui/src/main/resources/ui/src/styles/antd-master.less

@@ -18,3 +18,4 @@
 @import url('./components/table.less');
 @import url('./components/toggle-switch.less');
 @import url('./components/button.less');
+@import url('./components/card.less');

openmetadata-ui/src/main/resources/ui/src/styles/components/card.less

@@ -0,0 +1,4 @@
+.ant-card {
+  box-shadow: @card-shadow;
+  border: 1px #dde3ea solid;
+}

openmetadata-ui/src/main/resources/ui/src/styles/x-master.css

@@ -1060,14 +1060,6 @@ code {
   background-color: #dbd1f9;
 }
 
-.ant-select-single:not(.ant-select-customize-input) .ant-select-selector {
-  padding: 0px 4px;
-}
-
-.ant-select-single .ant-select-selector .ant-select-selection-search {
-  left: 4px;
-}
-
 @media only screen and (max-width: 1440px) {
   #left-panel {
     width: 284px;