fix(ui): API endpoints to fetch elastic search data on settings page. (#10946)

* Made changes to use the newly created APIs for fetching elasticSearch re-indexing data

* DOcs Fix and add validation

* Fixed APIs to fetch the batchJob reIndex data
worked on comments

---------

Co-authored-by: Sriharsha Chintalapani <harshach@users.noreply.github.com>
Co-authored-by: mohitdeuex <mohit.y@deuexsolutions.com>

openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java

@@ -418,40 +418,34 @@ public class SearchResource {
     return Response.status(OK).entity(response).build();
   }
 
-  @POST
+  @GET
-  @Path("/reindex")
+  @Path("/reindex/latest")
   @Operation(
-      operationId = "reindexEntities",
+      operationId = "getLatestReindexBatchJob",
-      summary = "Reindex entities",
+      summary = "Get Latest Reindexing Batch Job",
-      description = "Reindex Elastic Search Entities",
+      description = "Fetches the Latest Reindexing Job",
       responses = {
         @ApiResponse(responseCode = "200", description = "Success"),
-        @ApiResponse(responseCode = "404", description = "Bot for instance {id} is not found")
+        @ApiResponse(responseCode = "404", description = "No Job Found")
       })
-  public Response reindexEntities(
+  public Response reindexLatestJob(@Context UriInfo uriInfo, @Context SecurityContext securityContext)
-      @Context UriInfo uriInfo,
+      throws IOException {
-      @Context SecurityContext securityContext,
+    // Only admins  can issue a reindex request
-      @Valid CreateEventPublisherJob createRequest) {
     authorizer.authorizeAdmin(securityContext);
-    return Response.status(Response.Status.CREATED)
+    return Response.status(Response.Status.OK).entity(ReIndexingHandler.getInstance().getLatestJob()).build();
-        .entity(
-            ReIndexingHandler.getInstance()
-                .createReindexingJob(securityContext.getUserPrincipal().getName(), createRequest))
-        .build();
   }
 
   @GET
   @Path("/reindex/stream/status")
   @Operation(
-      operationId = "getStreamJobCurrentStatus",
+      operationId = "getStreamJobStatus",
-      summary = "Get Stream Job Current Status",
+      summary = "Get Stream Job Latest Status",
-      description = "Reindex all job last status",
+      description = "Stream Job Status",
       responses = {
         @ApiResponse(responseCode = "200", description = "Success"),
-        @ApiResponse(responseCode = "404", description = "Run model {runMode} is not found")
+        @ApiResponse(responseCode = "404", description = "Status not found")
       })
-  public Response reindexAllJobLastStatus(
+  public Response reindexAllJobLastStatus(@Context UriInfo uriInfo, @Context SecurityContext securityContext)
-      @Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("runMode") String runMode)
       throws IOException {
     // Only admins  can issue a reindex request
     authorizer.authorizeAdmin(securityContext);
@@ -466,30 +460,11 @@ public class SearchResource {
     return Response.status(Response.Status.NOT_FOUND).entity("No Last Run.").build();
   }
 
-  @GET
-  @Path("/reindex/latest")
-  @Operation(
-      operationId = "getReindexLatestJob",
-      summary = "Get last reindex job status",
-      tags = "search",
-      description = "Last Reindex job last status",
-      responses = {
-        @ApiResponse(responseCode = "200", description = "Success"),
-        @ApiResponse(responseCode = "404", description = "Run model {runMode} is not found")
-      })
-  public Response reindexLatestJob(@Context UriInfo uriInfo, @Context SecurityContext securityContext)
-      throws IOException {
-    // Only admins  can issue a reindex request
-    authorizer.authorizeAdmin(securityContext);
-    return Response.status(Response.Status.OK).entity(ReIndexingHandler.getInstance().getLatestJob()).build();
-  }
-
   @GET
   @Path("/reindex/{jobId}")
   @Operation(
-      operationId = "getReindexJobId",
+      operationId = "getBatchReindexBatchJobWithId",
-      summary = "Get reindex job with Id",
+      summary = "Get Batch Reindexing Job with Id",
-      tags = "search",
       description = "Get reindex job with Id",
       responses = {
         @ApiResponse(responseCode = "200", description = "Success"),
@@ -508,10 +483,10 @@ public class SearchResource {
   @GET
   @Path("/reindex")
   @Operation(
-      operationId = "getAllReindexJob",
+      operationId = "getAllReindexBatchJobs",
-      summary = "Get all reindex job",
+      summary = "Get all reindex batch jobs",
       tags = "search",
-      description = "Get all reindex",
+      description = "Get all reindex batch jobs",
       responses = {
         @ApiResponse(responseCode = "200", description = "Success"),
         @ApiResponse(responseCode = "404", description = "Not found")
@@ -523,6 +498,28 @@ public class SearchResource {
     return Response.status(Response.Status.OK).entity(ReIndexingHandler.getInstance().getAllJobs()).build();
   }
 
+  @POST
+  @Path("/reindex")
+  @Operation(
+      operationId = "runBatchReindexing",
+      summary = "Run Batch Reindexing",
+      description = "Reindex Elastic Search Reindexing Entities",
+      responses = {
+        @ApiResponse(responseCode = "200", description = "Success"),
+        @ApiResponse(responseCode = "404", description = "Bot for instance {id} is not found")
+      })
+  public Response reindexEntities(
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext,
+      @Valid CreateEventPublisherJob createRequest) {
+    authorizer.authorizeAdmin(securityContext);
+    return Response.status(Response.Status.CREATED)
+        .entity(
+            ReIndexingHandler.getInstance()
+                .createReindexingJob(securityContext.getUserPrincipal().getName(), createRequest))
+        .build();
+  }
+
   private SearchSourceBuilder buildAggregateSearchBuilder(String query, int from, int size) {
     QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query).lenient(true);
     SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);

openmetadata-service/src/main/java/org/openmetadata/service/util/ReIndexingHandler.java

@@ -18,9 +18,11 @@ import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ArrayBlockingQueue;
@@ -36,8 +38,10 @@ import org.openmetadata.schema.api.CreateEventPublisherJob;
 import org.openmetadata.schema.system.EventPublisherJob;
 import org.openmetadata.schema.system.Failure;
 import org.openmetadata.schema.system.Stats;
+import org.openmetadata.service.Entity;
 import org.openmetadata.service.elasticsearch.ElasticSearchIndexDefinition;
 import org.openmetadata.service.jdbi3.CollectionDAO;
+import org.openmetadata.service.workflows.searchIndex.ReindexingUtil;
 import org.openmetadata.service.workflows.searchIndex.SearchIndexWorkflow;
 
 @Slf4j
@@ -80,6 +84,9 @@ public class ReIndexingHandler {
     // Remove jobs in case they are completed
     clearCompletedJobs();
 
+    // validate current job
+    validateJob(createReindexingJob);
+
     // Create new Task
     if (taskQueue.size() >= 5) {
       throw new RuntimeException("Cannot create new Reindexing Jobs. There are pending jobs.");
@@ -125,6 +132,23 @@ public class ReIndexingHandler {
                     && entry.getValue().getJobData().getStatus() != EventPublisherJob.Status.RUNNING);
   }
 
+  private void validateJob(CreateEventPublisherJob job) {
+    Objects.requireNonNull(job);
+    Set<String> storedEntityList = new HashSet<>(Entity.getEntityList());
+    if (job.getEntities().size() > 0) {
+      job.getEntities()
+          .forEach(
+              (entityType) -> {
+                if (!storedEntityList.contains(entityType) && !ReindexingUtil.isDataInsightIndex(entityType)) {
+                  throw new IllegalArgumentException(
+                      String.format("Entity Type : %s is not a valid Entity", entityType));
+                }
+              });
+    } else {
+      throw new IllegalArgumentException("Entities cannot be Empty");
+    }
+  }
+
   public void removeCompletedJob(UUID jobId) {
     REINDEXING_JOB_MAP.remove(jobId);
   }

openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.css

@@ -29,6 +29,13 @@
   margin: auto;
 }
 
+.loader.loader-x-sm {
+  width: 14px;
+  height: 14px;
+  border-width: 1.5px;
+  margin: auto;
+}
+
 .loader.loader-success {
   border-left-color: #51c41a;
 }

openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.tsx

@@ -16,7 +16,7 @@ import React, { CSSProperties, FunctionComponent } from 'react';
 import './Loader.css';
 
 type Props = {
-  size?: 'default' | 'small';
+  size?: 'default' | 'small' | 'x-small';
   type?: 'default' | 'success' | 'error' | 'white';
   className?: string;
   style?: CSSProperties;
@@ -34,6 +34,11 @@ const Loader: FunctionComponent<Props> = ({
       classes += ' loader-sm';
 
       break;
+    case 'x-small':
+      classes += ' loader-x-sm';
+
+      break;
+
     default:
       break;
   }

openmetadata-ui/src/main/resources/ui/src/constants/elasticsearch.constant.ts

@@ -77,7 +77,6 @@ export const ELASTIC_SEARCH_INDEX_ENTITIES = [
 export const ELASTIC_SEARCH_INITIAL_VALUES = {
   entities: ['all'],
   batchSize: 100,
-  flushIntervalInSec: 30,
   recreateIndex: true,
   searchIndexMappingLanguage: SearchIndexMappingLanguage.En,
 };

openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/ElasticSearchReIndexModal.component.tsx

@@ -75,16 +75,6 @@ const ReIndexAllModal = ({
             treeData={ENTITY_TREE_OPTIONS}
           />
         </Form.Item>
-        <Form.Item
-          label={t('label.flush-interval-secs')}
-          name="flushIntervalInSec">
-          <Input
-            data-testid="flush-interval-in-sec"
-            placeholder={t('label.enter-entity', {
-              entity: t('label.second-plural'),
-            })}
-          />
-        </Form.Item>
 
         <Form.Item label={`${t('label.batch-size')}:`} name="batchSize">
           <Input

openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/ElasticSearchReIndexPage.component.tsx

@@ -25,7 +25,8 @@ import { isEmpty, isEqual, startCase } from 'lodash';
 import React, { useEffect, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import {
-  getAllReIndexStatus,
+  getBatchJobReIndexStatus,
+  getStreamJobReIndexStatus,
   reIndexByPublisher,
 } from 'rest/elasticSearchReIndexAPI';
 import { SOCKET_EVENTS } from '../../constants/constants';
@@ -40,7 +41,6 @@ import {
   getEventPublisherStatusText,
   getStatusResultBadgeIcon,
 } from '../../utils/EventPublisherUtils';
-import SVGIcons from '../../utils/SvgUtils';
 import { getDateTimeByTimeStampWithZone } from '../../utils/TimeUtils';
 import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils';
 import './ElasticSearchReIndex.style.less';
@@ -63,7 +63,7 @@ const ElasticSearchIndexPage = () => {
   const fetchBatchReIndexedData = async () => {
     try {
       setBatchLoading(true);
-      const response = await getAllReIndexStatus(RunMode.Batch);
+      const response = await getBatchJobReIndexStatus();
 
       setBatchJobData(response);
     } catch {
@@ -76,7 +76,7 @@ const ElasticSearchIndexPage = () => {
   const fetchStreamReIndexedData = async () => {
     try {
       setStreamLoading(true);
-      const response = await getAllReIndexStatus(RunMode.Stream);
+      const response = await getStreamJobReIndexStatus();
 
       setStreamJobData(response);
     } catch {
@@ -191,24 +191,15 @@ const ElasticSearchIndexPage = () => {
                         <span className="text-grey-muted">{`${t(
                           'label.status'
                         )}:`}</span>
-                        <span className="m-l-xs">
+
-                          <Space size={8}>
+                        <Space align="center" className="m-l-xs" size={8}>
-                            {batchJobData?.status && (
+                          {getStatusResultBadgeIcon(batchJobData?.status)}
-                              <SVGIcons
+                          <span>
-                                alt="result"
+                            {getEventPublisherStatusText(
-                                className="w-4"
+                              batchJobData?.status
-                                icon={getStatusResultBadgeIcon(
+                            ) || '--'}
-                                  batchJobData?.status
+                          </span>
-                                )}
+                        </Space>
-                              />
-                            )}
-                            <span>
-                              {getEventPublisherStatusText(
-                                batchJobData?.status
-                              ) || '--'}
-                            </span>
-                          </Space>
-                        </span>
                       </div>
                       <Divider type="vertical" />
                       <div className="flex">
@@ -232,15 +223,13 @@ const ElasticSearchIndexPage = () => {
                               <Badge
                                 className="request-badge success"
                                 count={
-                                  batchJobData?.stats?.jobStats
+                                  batchJobData?.stats?.jobStats?.successRecords
-                                    ?.totalSuccessRecords
                                 }
                                 overflowCount={99999999}
                                 title={`${t('label.entity-index', {
                                   entity: t('label.success'),
                                 })}: ${
-                                  batchJobData?.stats?.jobStats
+                                  batchJobData?.stats?.jobStats?.successRecords
-                                    ?.totalSuccessRecords
                                 }`}
                               />
 
@@ -248,15 +237,13 @@ const ElasticSearchIndexPage = () => {
                                 showZero
                                 className="request-badge failed"
                                 count={
-                                  batchJobData?.stats?.jobStats
+                                  batchJobData?.stats?.jobStats?.failedRecords
-                                    ?.totalFailedRecords
                                 }
                                 overflowCount={99999999}
                                 title={`${t('label.entity-index', {
                                   entity: t('label.failed'),
                                 })}: ${
-                                  batchJobData?.stats?.jobStats
+                                  batchJobData?.stats?.jobStats?.failedRecords
-                                    ?.totalFailedRecords
                                 }`}
                               />
                             </Space>
@@ -346,7 +333,7 @@ const ElasticSearchIndexPage = () => {
                 title={t('label.elasticsearch')}>
                 <Row gutter={[16, 8]}>
                   <Col span={24}>
-                    <Space direction="horizontal" size={16}>
+                    <Space direction="horizontal" size={0}>
                       <div className="flex">
                         <span className="text-grey-muted">{`${t(
                           'label.mode'
@@ -355,30 +342,21 @@ const ElasticSearchIndexPage = () => {
                           {startCase(streamJobData?.runMode) || '--'}
                         </span>
                       </div>
+                      <Divider type="vertical" />
                       <div className="flex">
                         <span className="text-grey-muted">{`${t(
                           'label.status'
                         )}:`}</span>
-                        <span className="m-l-xs">
+                        <Space align="center" className="m-l-xs" size={8}>
-                          <Space size={8}>
+                          {getStatusResultBadgeIcon(streamJobData?.status)}
-                            {streamJobData?.status && (
+                          <span>
-                              <SVGIcons
+                            {getEventPublisherStatusText(
-                                alt="result"
+                              streamJobData?.status
-                                className="w-4"
+                            ) || '--'}
-                                icon={getStatusResultBadgeIcon(
+                          </span>
-                                  streamJobData?.status
+                        </Space>
-                                )}
-                              />
-                            )}
-                            <span>
-                              {getEventPublisherStatusText(
-                                streamJobData?.status
-                              ) || '--'}
-                            </span>
-                          </Space>
-                        </span>
                       </div>
-
+                      <Divider type="vertical" />
                       <div className="flex">
                         <span className="text-grey-muted">{`${t(
                           'label.last-updated'
@@ -391,6 +369,7 @@ const ElasticSearchIndexPage = () => {
                             : '--'}
                         </span>
                       </div>
+                      <Divider type="vertical" />
                       <div className="flex">
                         <span className="text-grey-muted">{`${t(
                           'label.last-failed-at'

openmetadata-ui/src/main/resources/ui/src/rest/elasticSearchReIndexAPI.ts

@@ -17,17 +17,22 @@ import { CreateEventPublisherJob } from '../generated/api/createEventPublisherJo
 import {
   EventPublisherJob,
   PublisherType,
-  RunMode,
 } from '../generated/system/eventPublisherJob';
 
-export const getAllReIndexStatus = async (mode: RunMode) => {
+export const getStreamJobReIndexStatus = async () => {
   const res = await axiosClient.get<EventPublisherJob>(
-    `/indexResource/reindex/status/${mode}`
+    `/search/reindex/stream/status`
   );
 
   return res.data;
 };
 
+export const getBatchJobReIndexStatus = async () => {
+  const res = await axiosClient.get<EventPublisherJob>(`search/reindex/latest`);
+
+  return res.data;
+};
+
 export const reIndexByPublisher = async (data: CreateEventPublisherJob) => {
   const payload = {
     ...data,
@@ -37,7 +42,7 @@ export const reIndexByPublisher = async (data: CreateEventPublisherJob) => {
   const res = await axiosClient.post<
     CreateEventPublisherJob,
     AxiosResponse<EventPublisherJob>
-  >('/indexResource/reindex', payload);
+  >('/search/reindex', payload);
 
   return res.data;
 };

openmetadata-ui/src/main/resources/ui/src/utils/EventPublisherUtils.tsx

@@ -11,26 +11,30 @@
  *  limitations under the License.
  */
 
+import Loader from 'components/Loader/Loader';
 import { Status } from 'generated/system/eventPublisherJob';
 import { t } from 'i18next';
-import { Icons } from './SvgUtils';
+import React from 'react';
+import { ReactComponent as IconFailBadge } from '../assets/svg/fail-badge.svg';
+import { ReactComponent as IconTaskOpen } from '../assets/svg/in-progress.svg';
+import { ReactComponent as IconSuccessBadge } from '../assets/svg/success-badge.svg';
 
-export const getStatusResultBadgeIcon = (status: string) => {
+export const getStatusResultBadgeIcon = (status?: string) => {
   switch (status) {
-    case Status.Running:
-    case Status.Started:
-    case Status.Active:
-      return Icons.TASK_OPEN;
-
     case Status.Completed:
-      return Icons.SUCCESS_BADGE;
+      return <IconSuccessBadge height={14} width={14} />;
 
     case Status.Failed:
     case Status.ActiveWithError:
-      return Icons.FAIL_BADGE;
+      return <IconFailBadge height={14} width={14} />;
 
+    case Status.Running:
+    case Status.Started:
+      return <Loader size="x-small" />;
+
+    case Status.Active:
     default:
-      return '';
+      return <IconTaskOpen height={14} width={14} />;
   }
 };