diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java index 62d8a4f67fc..a5aab45c391 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java @@ -418,40 +418,34 @@ public class SearchResource { return Response.status(OK).entity(response).build(); } - @POST - @Path("/reindex") + @GET + @Path("/reindex/latest") @Operation( - operationId = "reindexEntities", - summary = "Reindex entities", - description = "Reindex Elastic Search Entities", + operationId = "getLatestReindexBatchJob", + summary = "Get Latest Reindexing Batch Job", + description = "Fetches the Latest Reindexing Job", responses = { @ApiResponse(responseCode = "200", description = "Success"), - @ApiResponse(responseCode = "404", description = "Bot for instance {id} is not found") + @ApiResponse(responseCode = "404", description = "No Job Found") }) - public Response reindexEntities( - @Context UriInfo uriInfo, - @Context SecurityContext securityContext, - @Valid CreateEventPublisherJob createRequest) { + public Response reindexLatestJob(@Context UriInfo uriInfo, @Context SecurityContext securityContext) + throws IOException { + // Only admins can issue a reindex request authorizer.authorizeAdmin(securityContext); - return Response.status(Response.Status.CREATED) - .entity( - ReIndexingHandler.getInstance() - .createReindexingJob(securityContext.getUserPrincipal().getName(), createRequest)) - .build(); + return Response.status(Response.Status.OK).entity(ReIndexingHandler.getInstance().getLatestJob()).build(); } @GET @Path("/reindex/stream/status") @Operation( - operationId = "getStreamJobCurrentStatus", - summary = "Get Stream Job Current Status", - description = "Reindex all job last status", + operationId = "getStreamJobStatus", + summary = "Get Stream Job Latest Status", + description = "Stream Job Status", responses = { @ApiResponse(responseCode = "200", description = "Success"), - @ApiResponse(responseCode = "404", description = "Run model {runMode} is not found") + @ApiResponse(responseCode = "404", description = "Status not found") }) - public Response reindexAllJobLastStatus( - @Context UriInfo uriInfo, @Context SecurityContext securityContext, @PathParam("runMode") String runMode) + public Response reindexAllJobLastStatus(@Context UriInfo uriInfo, @Context SecurityContext securityContext) throws IOException { // Only admins can issue a reindex request authorizer.authorizeAdmin(securityContext); @@ -466,30 +460,11 @@ public class SearchResource { return Response.status(Response.Status.NOT_FOUND).entity("No Last Run.").build(); } - @GET - @Path("/reindex/latest") - @Operation( - operationId = "getReindexLatestJob", - summary = "Get last reindex job status", - tags = "search", - description = "Last Reindex job last status", - responses = { - @ApiResponse(responseCode = "200", description = "Success"), - @ApiResponse(responseCode = "404", description = "Run model {runMode} is not found") - }) - public Response reindexLatestJob(@Context UriInfo uriInfo, @Context SecurityContext securityContext) - throws IOException { - // Only admins can issue a reindex request - authorizer.authorizeAdmin(securityContext); - return Response.status(Response.Status.OK).entity(ReIndexingHandler.getInstance().getLatestJob()).build(); - } - @GET @Path("/reindex/{jobId}") @Operation( - operationId = "getReindexJobId", - summary = "Get reindex job with Id", - tags = "search", + operationId = "getBatchReindexBatchJobWithId", + summary = "Get Batch Reindexing Job with Id", description = "Get reindex job with Id", responses = { @ApiResponse(responseCode = "200", description = "Success"), @@ -508,10 +483,10 @@ public class SearchResource { @GET @Path("/reindex") @Operation( - operationId = "getAllReindexJob", - summary = "Get all reindex job", + operationId = "getAllReindexBatchJobs", + summary = "Get all reindex batch jobs", tags = "search", - description = "Get all reindex", + description = "Get all reindex batch jobs", responses = { @ApiResponse(responseCode = "200", description = "Success"), @ApiResponse(responseCode = "404", description = "Not found") @@ -523,6 +498,28 @@ public class SearchResource { return Response.status(Response.Status.OK).entity(ReIndexingHandler.getInstance().getAllJobs()).build(); } + @POST + @Path("/reindex") + @Operation( + operationId = "runBatchReindexing", + summary = "Run Batch Reindexing", + description = "Reindex Elastic Search Reindexing Entities", + responses = { + @ApiResponse(responseCode = "200", description = "Success"), + @ApiResponse(responseCode = "404", description = "Bot for instance {id} is not found") + }) + public Response reindexEntities( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Valid CreateEventPublisherJob createRequest) { + authorizer.authorizeAdmin(securityContext); + return Response.status(Response.Status.CREATED) + .entity( + ReIndexingHandler.getInstance() + .createReindexingJob(securityContext.getUserPrincipal().getName(), createRequest)) + .build(); + } + private SearchSourceBuilder buildAggregateSearchBuilder(String query, int from, int size) { QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(query).lenient(true); SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/ReIndexingHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/ReIndexingHandler.java index 3034fa06da0..7be1a25d231 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/ReIndexingHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/ReIndexingHandler.java @@ -18,9 +18,11 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; import java.util.Date; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; @@ -36,8 +38,10 @@ import org.openmetadata.schema.api.CreateEventPublisherJob; import org.openmetadata.schema.system.EventPublisherJob; import org.openmetadata.schema.system.Failure; import org.openmetadata.schema.system.Stats; +import org.openmetadata.service.Entity; import org.openmetadata.service.elasticsearch.ElasticSearchIndexDefinition; import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.workflows.searchIndex.ReindexingUtil; import org.openmetadata.service.workflows.searchIndex.SearchIndexWorkflow; @Slf4j @@ -80,6 +84,9 @@ public class ReIndexingHandler { // Remove jobs in case they are completed clearCompletedJobs(); + // validate current job + validateJob(createReindexingJob); + // Create new Task if (taskQueue.size() >= 5) { throw new RuntimeException("Cannot create new Reindexing Jobs. There are pending jobs."); @@ -125,6 +132,23 @@ public class ReIndexingHandler { && entry.getValue().getJobData().getStatus() != EventPublisherJob.Status.RUNNING); } + private void validateJob(CreateEventPublisherJob job) { + Objects.requireNonNull(job); + Set storedEntityList = new HashSet<>(Entity.getEntityList()); + if (job.getEntities().size() > 0) { + job.getEntities() + .forEach( + (entityType) -> { + if (!storedEntityList.contains(entityType) && !ReindexingUtil.isDataInsightIndex(entityType)) { + throw new IllegalArgumentException( + String.format("Entity Type : %s is not a valid Entity", entityType)); + } + }); + } else { + throw new IllegalArgumentException("Entities cannot be Empty"); + } + } + public void removeCompletedJob(UUID jobId) { REINDEXING_JOB_MAP.remove(jobId); } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.css b/openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.css index 8d7c9680713..d07900eb0f6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.css +++ b/openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.css @@ -29,6 +29,13 @@ margin: auto; } +.loader.loader-x-sm { + width: 14px; + height: 14px; + border-width: 1.5px; + margin: auto; +} + .loader.loader-success { border-left-color: #51c41a; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.tsx index d987d78e866..04141a3f8e4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Loader/Loader.tsx @@ -16,7 +16,7 @@ import React, { CSSProperties, FunctionComponent } from 'react'; import './Loader.css'; type Props = { - size?: 'default' | 'small'; + size?: 'default' | 'small' | 'x-small'; type?: 'default' | 'success' | 'error' | 'white'; className?: string; style?: CSSProperties; @@ -34,6 +34,11 @@ const Loader: FunctionComponent = ({ classes += ' loader-sm'; break; + case 'x-small': + classes += ' loader-x-sm'; + + break; + default: break; } diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/elasticsearch.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/elasticsearch.constant.ts index cff9bceb865..158cd53e448 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/elasticsearch.constant.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/elasticsearch.constant.ts @@ -77,7 +77,6 @@ export const ELASTIC_SEARCH_INDEX_ENTITIES = [ export const ELASTIC_SEARCH_INITIAL_VALUES = { entities: ['all'], batchSize: 100, - flushIntervalInSec: 30, recreateIndex: true, searchIndexMappingLanguage: SearchIndexMappingLanguage.En, }; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/ElasticSearchReIndexModal.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/ElasticSearchReIndexModal.component.tsx index 32987a9c5ac..4c0a64ff73f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/ElasticSearchReIndexModal.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/ElasticSearchIndexPage/ElasticSearchReIndexModal.component.tsx @@ -75,16 +75,6 @@ const ReIndexAllModal = ({ treeData={ENTITY_TREE_OPTIONS} /> - - - { const fetchBatchReIndexedData = async () => { try { setBatchLoading(true); - const response = await getAllReIndexStatus(RunMode.Batch); + const response = await getBatchJobReIndexStatus(); setBatchJobData(response); } catch { @@ -76,7 +76,7 @@ const ElasticSearchIndexPage = () => { const fetchStreamReIndexedData = async () => { try { setStreamLoading(true); - const response = await getAllReIndexStatus(RunMode.Stream); + const response = await getStreamJobReIndexStatus(); setStreamJobData(response); } catch { @@ -191,24 +191,15 @@ const ElasticSearchIndexPage = () => { {`${t( 'label.status' )}:`} - - - {batchJobData?.status && ( - - )} - - {getEventPublisherStatusText( - batchJobData?.status - ) || '--'} - - - + + + {getStatusResultBadgeIcon(batchJobData?.status)} + + {getEventPublisherStatusText( + batchJobData?.status + ) || '--'} + +
@@ -232,15 +223,13 @@ const ElasticSearchIndexPage = () => { @@ -248,15 +237,13 @@ const ElasticSearchIndexPage = () => { showZero className="request-badge failed" count={ - batchJobData?.stats?.jobStats - ?.totalFailedRecords + batchJobData?.stats?.jobStats?.failedRecords } overflowCount={99999999} title={`${t('label.entity-index', { entity: t('label.failed'), })}: ${ - batchJobData?.stats?.jobStats - ?.totalFailedRecords + batchJobData?.stats?.jobStats?.failedRecords }`} /> @@ -346,7 +333,7 @@ const ElasticSearchIndexPage = () => { title={t('label.elasticsearch')}> - +
{`${t( 'label.mode' @@ -355,30 +342,21 @@ const ElasticSearchIndexPage = () => { {startCase(streamJobData?.runMode) || '--'}
+
{`${t( 'label.status' )}:`} - - - {streamJobData?.status && ( - - )} - - {getEventPublisherStatusText( - streamJobData?.status - ) || '--'} - - - + + {getStatusResultBadgeIcon(streamJobData?.status)} + + {getEventPublisherStatusText( + streamJobData?.status + ) || '--'} + +
- +
{`${t( 'label.last-updated' @@ -391,6 +369,7 @@ const ElasticSearchIndexPage = () => { : '--'}
+
{`${t( 'label.last-failed-at' diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/elasticSearchReIndexAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/elasticSearchReIndexAPI.ts index 4003e0a810b..853ab759659 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/elasticSearchReIndexAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/elasticSearchReIndexAPI.ts @@ -17,17 +17,22 @@ import { CreateEventPublisherJob } from '../generated/api/createEventPublisherJo import { EventPublisherJob, PublisherType, - RunMode, } from '../generated/system/eventPublisherJob'; -export const getAllReIndexStatus = async (mode: RunMode) => { +export const getStreamJobReIndexStatus = async () => { const res = await axiosClient.get( - `/indexResource/reindex/status/${mode}` + `/search/reindex/stream/status` ); return res.data; }; +export const getBatchJobReIndexStatus = async () => { + const res = await axiosClient.get(`search/reindex/latest`); + + return res.data; +}; + export const reIndexByPublisher = async (data: CreateEventPublisherJob) => { const payload = { ...data, @@ -37,7 +42,7 @@ export const reIndexByPublisher = async (data: CreateEventPublisherJob) => { const res = await axiosClient.post< CreateEventPublisherJob, AxiosResponse - >('/indexResource/reindex', payload); + >('/search/reindex', payload); return res.data; }; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EventPublisherUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/EventPublisherUtils.tsx similarity index 70% rename from openmetadata-ui/src/main/resources/ui/src/utils/EventPublisherUtils.ts rename to openmetadata-ui/src/main/resources/ui/src/utils/EventPublisherUtils.tsx index c1503e4af36..a821ba01f7b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EventPublisherUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EventPublisherUtils.tsx @@ -11,26 +11,30 @@ * limitations under the License. */ +import Loader from 'components/Loader/Loader'; import { Status } from 'generated/system/eventPublisherJob'; import { t } from 'i18next'; -import { Icons } from './SvgUtils'; +import React from 'react'; +import { ReactComponent as IconFailBadge } from '../assets/svg/fail-badge.svg'; +import { ReactComponent as IconTaskOpen } from '../assets/svg/in-progress.svg'; +import { ReactComponent as IconSuccessBadge } from '../assets/svg/success-badge.svg'; -export const getStatusResultBadgeIcon = (status: string) => { +export const getStatusResultBadgeIcon = (status?: string) => { switch (status) { - case Status.Running: - case Status.Started: - case Status.Active: - return Icons.TASK_OPEN; - case Status.Completed: - return Icons.SUCCESS_BADGE; + return ; case Status.Failed: case Status.ActiveWithError: - return Icons.FAIL_BADGE; + return ; + case Status.Running: + case Status.Started: + return ; + + case Status.Active: default: - return ''; + return ; } };