fix(ui): API endpoints to fetch elastic search data on settings page. (#10946)

* Made changes to use the newly created APIs for fetching elasticSearch re-indexing data * DOcs Fix and add validation * Fixed APIs to fetch the batchJob reIndex data worked on comments --------- Co-authored-by: Sriharsha Chintalapani <harshach@users.noreply.github.com> Co-authored-by: mohitdeuex <mohit.y@deuexsolutions.com>
2025-10-31 02:29:03 +00:00 · 2023-04-07 11:53:51 +05:30 · 2023-04-07 11:53:51 +05:30 · d112b40a3f
commit d112b40a3f
parent c997d8c80b
9 changed files with 131 additions and 121 deletions
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java
@ -418,40 +418,34 @@ public class SearchResource {
    return Response.status(OK).entity(response).build();
  }

-  @POST
-  @Path("/reindex")
+  @GET
+  @Path("/reindex/latest")
  @Operation(
-      operationId = "reindexEntities",
-      summary = "Reindex entities",
-      description = "Reindex Elastic Search Entities",
+      operationId = "getLatestReindexBatchJob",
+      summary = "Get Latest Reindexing Batch Job",
+      description = "Fetches the Latest Reindexing Job",
      responses = {
        @ApiResponse(responseCode = "200", description = "Success"),
-        @ApiResponse(responseCode = "404", description = "Bot for instance {id} is not found")
+        @ApiResponse(responseCode = "404", description = "No Job Found")
      })
-  public Response reindexEntities(
-      @Context UriInfo uriInfo,
-      @Context SecurityContext securityContext,
-      @Valid CreateEventPublisherJob createRequest) {
+  public Response reindexLatestJob(@Context UriInfo uriInfo, @Context SecurityContext securityContext)
+      throws IOException {
+    // Only admins  can issue a reindex request
    authorizer.authorizeAdmin(securityContext);
-    return Response.status(Response.Status.CREATED)
-        .entity(
-            ReIndexingHandler.getInstance()
-                .createReindexingJob(securityContext.getUserPrincipal().getName(), createRequest))
-        .build();
+    return Response.status(Response.Status.OK).entity(ReIndexingHandler.getInstance().getLatestJob()).build();
  }

  @GET
  @Path("/reindex/stream/status")
  @Operation(
-      operationId = "getStreamJobCurrentStatus",
-      summary = "Get Stream Job Current Status",
-      description = "Reindex all job last status",
+      operationId = "getStreamJobStatus",
+      summary = "Get Stream Job Latest Status",
+      description = "Stream Job Status",
      responses = {
        @ApiResponse(responseCode = "200", description = "Success"),
-        @ApiResponse(responseCode = "404", description = "Run model {runMode} is not found")
+        @ApiResponse(responseCode = "404", description = "Status not found")
      })
-  public Response reindexAllJobLastStatus(
-      @Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("runMode") String runMode)
+  public Response reindexAllJobLastStatus(@Context UriInfo uriInfo, @Context SecurityContext securityContext)
      throws IOException {
    // Only admins  can issue a reindex request
    authorizer.authorizeAdmin(securityContext);
@ -466,30 +460,11 @@ public class SearchResource {
    return Response.status(Response.Status.NOT_FOUND).entity("No Last Run.").build();
  }

-  @GET
-  @Path("/reindex/latest")
-  @Operation(
-      operationId = "getReindexLatestJob",
-      summary = "Get last reindex job status",
-      tags = "search",
-      description = "Last Reindex job last status",
-      responses = {
-        @ApiResponse(responseCode = "200", description = "Success"),
-        @ApiResponse(responseCode = "404", description = "Run model {runMode} is not found")
-      })
-  public Response reindexLatestJob(@Context UriInfo uriInfo, @Context SecurityContext securityContext)
-      throws IOException {
-    // Only admins  can issue a reindex request
-    authorizer.authorizeAdmin(securityContext);
-    return Response.status(Response.Status.OK).entity(ReIndexingHandler.getInstance().getLatestJob()).build();
-  }
-
  @GET
  @Path("/reindex/{jobId}")
  @Operation(
-      operationId = "getReindexJobId",
-      summary = "Get reindex job with Id",
-      tags = "search",
+      operationId = "getBatchReindexBatchJobWithId",
+      summary = "Get Batch Reindexing Job with Id",
      description = "Get reindex job with Id",
      responses = {
        @ApiResponse(responseCode = "200", description = "Success"),
@ -508,10 +483,10 @@ public class SearchResource {
  @GET
  @Path("/reindex")
  @Operation(
-      operationId = "getAllReindexJob",
-      summary = "Get all reindex job",
+      operationId = "getAllReindexBatchJobs",
+      summary = "Get all reindex batch jobs",
      tags = "search",
-      description = "Get all reindex",
+      description = "Get all reindex batch jobs",
      responses = {
        @ApiResponse(responseCode = "200", description = "Success"),
        @ApiResponse(responseCode = "404", description = "Not found")
@ -523,6 +498,28 @@ public class SearchResource {
    return Response.status(Response.Status.OK).entity(ReIndexingHandler.getInstance().getAllJobs()).build();
  }

+  @POST
+  @Path("/reindex")
+  @Operation(
+      operationId = "runBatchReindexing",
+      summary = "Run Batch Reindexing",
+      description = "Reindex Elastic Search Reindexing Entities",
+      responses = {
+        @ApiResponse(responseCode = "200", description = "Success"),
+        @ApiResponse(responseCode = "404", description = "Bot for instance {id} is not found")
+      })
+  public Response reindexEntities(
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext,
+      @Valid CreateEventPublisherJob createRequest) {
+    authorizer.authorizeAdmin(securityContext);
+    return Response.status(Response.Status.CREATED)
+        .entity(
+            ReIndexingHandler.getInstance()
+                .createReindexingJob(securityContext.getUserPrincipal().getName(), createRequest))
+        .build();
+  }
+
  private SearchSourceBuilder buildAggregateSearchBuilder(String query, int from, int size) {
    QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query).lenient(true);
    SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);
--- a/openmetadata-service/src/main/java/org/openmetadata/service/util/ReIndexingHandler.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/ReIndexingHandler.java
@ -18,9 +18,11 @@ import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.util.ArrayList;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ArrayBlockingQueue;
@ -36,8 +38,10 @@ import org.openmetadata.schema.api.CreateEventPublisherJob;
 import org.openmetadata.schema.system.EventPublisherJob;
 import org.openmetadata.schema.system.Failure;
 import org.openmetadata.schema.system.Stats;
+import org.openmetadata.service.Entity;
 import org.openmetadata.service.elasticsearch.ElasticSearchIndexDefinition;
 import org.openmetadata.service.jdbi3.CollectionDAO;
+import org.openmetadata.service.workflows.searchIndex.ReindexingUtil;
 import org.openmetadata.service.workflows.searchIndex.SearchIndexWorkflow;

@Slf4j
@ -80,6 +84,9 @@ public class ReIndexingHandler {
    // Remove jobs in case they are completed
    clearCompletedJobs();

+    // validate current job
+    validateJob(createReindexingJob);
+
    // Create new Task
    if (taskQueue.size() >= 5) {
      throw new RuntimeException("Cannot create new Reindexing Jobs. There are pending jobs.");
@ -125,6 +132,23 @@ public class ReIndexingHandler {
                    && entry.getValue().getJobData().getStatus() != EventPublisherJob.Status.RUNNING);
  }

+  private void validateJob(CreateEventPublisherJob job) {
+    Objects.requireNonNull(job);
+    Set<String> storedEntityList = new HashSet<>(Entity.getEntityList());
+    if (job.getEntities().size() > 0) {
+      job.getEntities()
+          .forEach(
+              (entityType) -> {
+                if (!storedEntityList.contains(entityType) && !ReindexingUtil.isDataInsightIndex(entityType)) {
+                  throw new IllegalArgumentException(
+                      String.format("Entity Type : %s is not a valid Entity", entityType));
+                }
+              });
+    } else {
+      throw new IllegalArgumentException("Entities cannot be Empty");
+    }
+  }
+
  public void removeCompletedJob(UUID jobId) {
    REINDEXING_JOB_MAP.remove(jobId);
  }
--- a/openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.css
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.css
@ -29,6 +29,13 @@
  margin: auto;
 }

+.loader.loader-x-sm {
+  width: 14px;
+  height: 14px;
+  border-width: 1.5px;
+  margin: auto;
+}
+
 .loader.loader-success {
  border-left-color: #51c41a;
 }
--- a/openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.tsx
@ -16,7 +16,7 @@ import React, { CSSProperties, FunctionComponent } from 'react';
 import './Loader.css';

 type Props = {
-  size?: 'default' | 'small';
+  size?: 'default' | 'small' | 'x-small';
  type?: 'default' | 'success' | 'error' | 'white';
  className?: string;
  style?: CSSProperties;
@ -34,6 +34,11 @@ const Loader: FunctionComponent<Props> = ({
      classes += ' loader-sm';

      break;
+    case 'x-small':
+      classes += ' loader-x-sm';
+
+      break;
+
    default:
      break;
  }
--- a/openmetadata-ui/src/main/resources/ui/src/constants/elasticsearch.constant.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/constants/elasticsearch.constant.ts
@ -77,7 +77,6 @@ export const ELASTIC_SEARCH_INDEX_ENTITIES = [
 export const ELASTIC_SEARCH_INITIAL_VALUES = {
  entities: ['all'],
  batchSize: 100,
-  flushIntervalInSec: 30,
  recreateIndex: true,
  searchIndexMappingLanguage: SearchIndexMappingLanguage.En,
 };
--- a/openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/ElasticSearchReIndexModal.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/ElasticSearchReIndexModal.component.tsx
@ -75,16 +75,6 @@ const ReIndexAllModal = ({
            treeData={ENTITY_TREE_OPTIONS}
          />
        </Form.Item>
-        <Form.Item
-          label={t('label.flush-interval-secs')}
-          name="flushIntervalInSec">
-          <Input
-            data-testid="flush-interval-in-sec"
-            placeholder={t('label.enter-entity', {
-              entity: t('label.second-plural'),
-            })}
-          />
-        </Form.Item>

        <Form.Item label={`${t('label.batch-size')}:`} name="batchSize">
          <Input
--- a/openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/ElasticSearchReIndexPage.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/ElasticSearchReIndexPage.component.tsx
@ -25,7 +25,8 @@ import { isEmpty, isEqual, startCase } from 'lodash';
 import React, { useEffect, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import {
-  getAllReIndexStatus,
+  getBatchJobReIndexStatus,
+  getStreamJobReIndexStatus,
  reIndexByPublisher,
 } from 'rest/elasticSearchReIndexAPI';
 import { SOCKET_EVENTS } from '../../constants/constants';
@ -40,7 +41,6 @@ import {
  getEventPublisherStatusText,
  getStatusResultBadgeIcon,
 } from '../../utils/EventPublisherUtils';
-import SVGIcons from '../../utils/SvgUtils';
 import { getDateTimeByTimeStampWithZone } from '../../utils/TimeUtils';
 import { showErrorToast, showSuccessToast } from '../../utils/ToastUtils';
 import './ElasticSearchReIndex.style.less';
@ -63,7 +63,7 @@ const ElasticSearchIndexPage = () => {
  const fetchBatchReIndexedData = async () => {
    try {
      setBatchLoading(true);
-      const response = await getAllReIndexStatus(RunMode.Batch);
+      const response = await getBatchJobReIndexStatus();

      setBatchJobData(response);
    } catch {
@ -76,7 +76,7 @@ const ElasticSearchIndexPage = () => {
  const fetchStreamReIndexedData = async () => {
    try {
      setStreamLoading(true);
-      const response = await getAllReIndexStatus(RunMode.Stream);
+      const response = await getStreamJobReIndexStatus();

      setStreamJobData(response);
    } catch {
@ -191,24 +191,15 @@ const ElasticSearchIndexPage = () => {
                        <span className="text-grey-muted">{`${t(
                          'label.status'
                        )}:`}</span>
-                        <span className="m-l-xs">
-                          <Space size={8}>
-                            {batchJobData?.status && (
-                              <SVGIcons
-                                alt="result"
-                                className="w-4"
-                                icon={getStatusResultBadgeIcon(
-                                  batchJobData?.status
-                                )}
-                              />
-                            )}
-                            <span>
-                              {getEventPublisherStatusText(
-                                batchJobData?.status
-                              ) || '--'}
-                            </span>
-                          </Space>
-                        </span>
+
+                        <Space align="center" className="m-l-xs" size={8}>
+                          {getStatusResultBadgeIcon(batchJobData?.status)}
+                          <span>
+                            {getEventPublisherStatusText(
+                              batchJobData?.status
+                            ) || '--'}
+                          </span>
+                        </Space>
                      </div>
                      <Divider type="vertical" />
                      <div className="flex">
@ -232,15 +223,13 @@ const ElasticSearchIndexPage = () => {
                              <Badge
                                className="request-badge success"
                                count={
-                                  batchJobData?.stats?.jobStats
-                                    ?.totalSuccessRecords
+                                  batchJobData?.stats?.jobStats?.successRecords
                                }
                                overflowCount={99999999}
                                title={`${t('label.entity-index', {
                                  entity: t('label.success'),
                                })}: ${
-                                  batchJobData?.stats?.jobStats
-                                    ?.totalSuccessRecords
+                                  batchJobData?.stats?.jobStats?.successRecords
                                }`}
                              />

@ -248,15 +237,13 @@ const ElasticSearchIndexPage = () => {
                                showZero
                                className="request-badge failed"
                                count={
-                                  batchJobData?.stats?.jobStats
-                                    ?.totalFailedRecords
+                                  batchJobData?.stats?.jobStats?.failedRecords
                                }
                                overflowCount={99999999}
                                title={`${t('label.entity-index', {
                                  entity: t('label.failed'),
                                })}: ${
-                                  batchJobData?.stats?.jobStats
-                                    ?.totalFailedRecords
+                                  batchJobData?.stats?.jobStats?.failedRecords
                                }`}
                              />
                            </Space>
@ -346,7 +333,7 @@ const ElasticSearchIndexPage = () => {
                title={t('label.elasticsearch')}>
                <Row gutter={[16, 8]}>
                  <Col span={24}>
-                    <Space direction="horizontal" size={16}>
+                    <Space direction="horizontal" size={0}>
                      <div className="flex">
                        <span className="text-grey-muted">{`${t(
                          'label.mode'
@ -355,30 +342,21 @@ const ElasticSearchIndexPage = () => {
                          {startCase(streamJobData?.runMode) || '--'}
                        </span>
                      </div>
+                      <Divider type="vertical" />
                      <div className="flex">
                        <span className="text-grey-muted">{`${t(
                          'label.status'
                        )}:`}</span>
-                        <span className="m-l-xs">
-                          <Space size={8}>
-                            {streamJobData?.status && (
-                              <SVGIcons
-                                alt="result"
-                                className="w-4"
-                                icon={getStatusResultBadgeIcon(
-                                  streamJobData?.status
-                                )}
-                              />
-                            )}
-                            <span>
-                              {getEventPublisherStatusText(
-                                streamJobData?.status
-                              ) || '--'}
-                            </span>
-                          </Space>
-                        </span>
+                        <Space align="center" className="m-l-xs" size={8}>
+                          {getStatusResultBadgeIcon(streamJobData?.status)}
+                          <span>
+                            {getEventPublisherStatusText(
+                              streamJobData?.status
+                            ) || '--'}
+                          </span>
+                        </Space>
                      </div>
-
+                      <Divider type="vertical" />
                      <div className="flex">
                        <span className="text-grey-muted">{`${t(
                          'label.last-updated'
@ -391,6 +369,7 @@ const ElasticSearchIndexPage = () => {
                            : '--'}
                        </span>
                      </div>
+                      <Divider type="vertical" />
                      <div className="flex">
                        <span className="text-grey-muted">{`${t(
                          'label.last-failed-at'
--- a/openmetadata-ui/src/main/resources/ui/src/rest/elasticSearchReIndexAPI.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/rest/elasticSearchReIndexAPI.ts
@ -17,17 +17,22 @@ import { CreateEventPublisherJob } from '../generated/api/createEventPublisherJo
 import {
  EventPublisherJob,
  PublisherType,
-  RunMode,
 } from '../generated/system/eventPublisherJob';

-export const getAllReIndexStatus = async (mode: RunMode) => {
+export const getStreamJobReIndexStatus = async () => {
  const res = await axiosClient.get<EventPublisherJob>(
-    `/indexResource/reindex/status/${mode}`
+    `/search/reindex/stream/status`
  );

  return res.data;
 };

+export const getBatchJobReIndexStatus = async () => {
+  const res = await axiosClient.get<EventPublisherJob>(`search/reindex/latest`);
+
+  return res.data;
+};
+
 export const reIndexByPublisher = async (data: CreateEventPublisherJob) => {
  const payload = {
    ...data,
@ -37,7 +42,7 @@ export const reIndexByPublisher = async (data: CreateEventPublisherJob) => {
  const res = await axiosClient.post<
    CreateEventPublisherJob,
    AxiosResponse<EventPublisherJob>
-  >('/indexResource/reindex', payload);
+  >('/search/reindex', payload);

  return res.data;
 };
--- a/openmetadata-ui/src/main/resources/ui/src/utils/EventPublisherUtils.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/EventPublisherUtils.tsx
@ -11,26 +11,30 @@
 *  limitations under the License.
 */

+import Loader from 'components/Loader/Loader';
 import { Status } from 'generated/system/eventPublisherJob';
 import { t } from 'i18next';
-import { Icons } from './SvgUtils';
+import React from 'react';
+import { ReactComponent as IconFailBadge } from '../assets/svg/fail-badge.svg';
+import { ReactComponent as IconTaskOpen } from '../assets/svg/in-progress.svg';
+import { ReactComponent as IconSuccessBadge } from '../assets/svg/success-badge.svg';

-export const getStatusResultBadgeIcon = (status: string) => {
+export const getStatusResultBadgeIcon = (status?: string) => {
  switch (status) {
-    case Status.Running:
-    case Status.Started:
-    case Status.Active:
-      return Icons.TASK_OPEN;
-
    case Status.Completed:
-      return Icons.SUCCESS_BADGE;
+      return <IconSuccessBadge height={14} width={14} />;

    case Status.Failed:
    case Status.ActiveWithError:
-      return Icons.FAIL_BADGE;
+      return <IconFailBadge height={14} width={14} />;

+    case Status.Running:
+    case Status.Started:
+      return <Loader size="x-small" />;
+
+    case Status.Active:
    default:
-      return '';
+      return <IconTaskOpen height={14} width={14} />;
  }
 };