yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-15 04:26:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make Export CSV Async API, websocket to push data back (#18497)

Browse Source 
* Make Export CSV Async API, websocket to push data back

* ui: implement csvExportChannel to receive entity export data via websocket

* ui: update flow to show dialogbox with the response message and once data is available download the file

* minor: remove unused code

* ui: show inline banner for success and error message

* remove unused locale keys

* ui: clear existing csvexport job on sucess

* fix failing UserResourceTest

---------

Co-authored-by: Sachin Chaurasiya <sachinchaurasiyachotey87@gmail.com>
Co-authored-by: sonikashah <sonikashah94@gmail.com>
This commit is contained in:
Sriharsha Chintalapani 2024-11-05 05:02:41 -08:00 committed by GitHub
parent a6b5fdc0c2
commit b763091bf6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
28 changed files with 441 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
openmetadata-service/pom.xml
View File

@ -27,6 +27,7 @@
<pac4j.version>5.7.0</pac4j.version>
<maven-javadoc-plugin.version>3.6.0</maven-javadoc-plugin.version>
<maven-source-plugin.version>3.3.1</maven-source-plugin.version>
<socket.io-client.version>2.1.1</socket.io-client.version>
</properties>
<dependencyManagement>
@ -367,6 +368,13 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.socket</groupId>
<artifactId>socket.io-client</artifactId>
<version>${socket.io-client.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>

26
openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java
View File

@ -13,6 +13,7 @@
package org.openmetadata.service.resources;
import static javax.ws.rs.client.Entity.entity;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.schema.type.EventType.ENTITY_CREATED;
@ -27,7 +28,9 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import javax.json.JsonPatch;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
@ -52,6 +55,8 @@ import org.openmetadata.service.security.policyevaluator.CreateResourceContext;
import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.security.policyevaluator.ResourceContext;
import org.openmetadata.service.security.policyevaluator.ResourceContextInterface;
import org.openmetadata.service.util.AsyncService;
import org.openmetadata.service.util.CSVExportResponse;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.RestUtil;
@ -59,6 +64,7 @@ import org.openmetadata.service.util.RestUtil.DeleteResponse;
import org.openmetadata.service.util.RestUtil.PatchResponse;
import org.openmetadata.service.util.RestUtil.PutResponse;
import org.openmetadata.service.util.ResultList;
import org.openmetadata.service.util.WebsocketNotificationHandler;
@Slf4j
public abstract class EntityResource<T extends EntityInterface, K extends EntityRepository<T>> {
@ -376,11 +382,27 @@ public abstract class EntityResource<T extends EntityInterface, K extends Entity
return response.toResponse();
}
public String exportCsvInternal(SecurityContext securityContext, String name) throws IOException {
public Response exportCsvInternal(SecurityContext securityContext, String name)
throws IOException {
OperationContext operationContext =
new OperationContext(entityType, MetadataOperation.VIEW_ALL);
authorizer.authorize(securityContext, operationContext, getResourceContextByName(name));
return repository.exportToCsv(name, securityContext.getUserPrincipal().getName());
String jobId = UUID.randomUUID().toString();
ExecutorService executorService = AsyncService.getInstance().getExecutorService();
executorService.submit(
() -> {
try {
String csvData =
repository.exportToCsv(name, securityContext.getUserPrincipal().getName());
WebsocketNotificationHandler.sendCsvExportCompleteNotification(
jobId, securityContext, csvData);
} catch (Exception e) {
WebsocketNotificationHandler.sendCsvExportFailedNotification(
jobId, securityContext, e.getMessage());
}
});
CSVExportResponse response = new CSVExportResponse(jobId, "Export initiated successfully.");
return Response.accepted().entity(response).type(MediaType.APPLICATION_JSON).build();
}
protected CsvImportResult importCsvInternal(

7
openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java
View File

@ -65,6 +65,7 @@ import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.util.CSVExportResponse;
import org.openmetadata.service.util.ResultList;
@Path("/v1/databases")
@ -423,7 +424,7 @@ public class DatabaseResource extends EntityResource<Database, DatabaseRepositor
@GET
@Path("/name/{name}/export")
@Produces(MediaType.TEXT_PLAIN)
@Produces(MediaType.APPLICATION_JSON)
@Valid
@Operation(
operationId = "exportDatabase",
@ -435,9 +436,9 @@ public class DatabaseResource extends EntityResource<Database, DatabaseRepositor
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = String.class)))
schema = @Schema(implementation = CSVExportResponse.class)))
})
public String exportCsv(
public Response exportCsv(
@Context SecurityContext securityContext,
@Parameter(description = "Name of the Database", schema = @Schema(type = "string"))
@PathParam("name")

7
openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java
View File

@ -62,6 +62,7 @@ import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.util.CSVExportResponse;
import org.openmetadata.service.util.ResultList;
@Path("/v1/databaseSchemas")
@ -395,7 +396,7 @@ public class DatabaseSchemaResource
@GET
@Path("/name/{name}/export")
@Produces(MediaType.TEXT_PLAIN)
@Produces(MediaType.APPLICATION_JSON)
@Valid
@Operation(
operationId = "exportDatabaseSchema",
@ -407,9 +408,9 @@ public class DatabaseSchemaResource
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = String.class)))
schema = @Schema(implementation = CSVExportResponse.class)))
})
public String exportCsv(
public Response exportCsv(
@Context SecurityContext securityContext,
@Parameter(description = "Name of the Database schema", schema = @Schema(type = "string"))
@PathParam("name")

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java
View File

@ -468,7 +468,7 @@ public class TableResource extends EntityResource<Table, TableRepository> {
mediaType = "application/json",
schema = @Schema(implementation = String.class)))
})
public String exportCsv(
public Response exportCsv(
@Context SecurityContext securityContext,
@Parameter(description = "Name of the table", schema = @Schema(type = "string"))
@PathParam("name")

5
openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java
View File

@ -63,6 +63,7 @@ import org.openmetadata.service.limits.Limits;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.util.CSVExportResponse;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ResultList;
@ -513,9 +514,9 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = String.class)))
schema = @Schema(implementation = CSVExportResponse.class)))
})
public String exportCsv(
public Response exportCsv(
@Context SecurityContext securityContext,
@Parameter(description = "Name of the glossary", schema = @Schema(type = "string"))
@PathParam("name")

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java
View File

@ -456,7 +456,7 @@ public class DatabaseServiceResource
mediaType = "application/json",
schema = @Schema(implementation = String.class)))
})
public String exportCsv(
public Response exportCsv(
@Context SecurityContext securityContext,
@Parameter(description = "Name of the Database Service", schema = @Schema(type = "string"))
@PathParam("name")

7
openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java
View File

@ -71,6 +71,7 @@ import org.openmetadata.service.limits.Limits;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.util.CSVExportResponse;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ResultList;
@ -606,10 +607,10 @@ public class TeamResource extends EntityResource<Team, TeamRepository> {
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = String.class)))
schema = @Schema(implementation = CSVExportResponse.class)))
})
public String exportCsv(@Context SecurityContext securityContext, @PathParam("name") String name)
throws IOException {
public Response exportCsv(
@Context SecurityContext securityContext, @PathParam("name") String name) throws IOException {
return exportCsvInternal(securityContext, name);
}

5
openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java
View File

@ -150,6 +150,7 @@ import org.openmetadata.service.security.mask.PIIMasker;
import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.security.policyevaluator.ResourceContext;
import org.openmetadata.service.security.saml.JwtTokenCacheManager;
import org.openmetadata.service.util.CSVExportResponse;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.JsonUtils;
@ -1404,9 +1405,9 @@ public class UserResource extends EntityResource<User, UserRepository> {
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = String.class)))
schema = @Schema(implementation = CSVExportResponse.class)))
})
public String exportUsersCsv(
public Response exportUsersCsv(
@Context SecurityContext securityContext,
@Parameter(
description = "Name of the team to under which the users are imported to",

1
openmetadata-service/src/main/java/org/openmetadata/service/socket/WebSocketManager.java
View File

@ -27,6 +27,7 @@ public class WebSocketManager {
public static final String JOB_STATUS_BROADCAST_CHANNEL = "jobStatus";
public static final String MENTION_CHANNEL = "mentionChannel";
public static final String ANNOUNCEMENT_CHANNEL = "announcementChannel";
public static final String CSV_EXPORT_CHANNEL = "csvExportChannel";
@Getter
private final Map<UUID, Map<String, SocketIoSocket>> activityFeedEndpoints =

44
openmetadata-service/src/main/java/org/openmetadata/service/util/AsyncService.java Normal file
View File

@ -0,0 +1,44 @@
package org.openmetadata.service.util;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
public class AsyncService {
private static AsyncService instance;
private final ExecutorService executorService;
private AsyncService() {
ThreadFactory threadFactory =
new ThreadFactory() {
private final AtomicInteger threadNumber = new AtomicInteger(1);
private final String namePrefix = "AsyncServicePool-Thread-";
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r, namePrefix + threadNumber.getAndIncrement());
if (t.isDaemon()) t.setDaemon(false);
if (t.getPriority() != Thread.NORM_PRIORITY) t.setPriority(Thread.NORM_PRIORITY);
return t;
}
};
executorService = Executors.newFixedThreadPool(20, threadFactory);
}
public static synchronized AsyncService getInstance() {
if (instance == null) {
instance = new AsyncService();
}
return instance;
}
public ExecutorService getExecutorService() {
return executorService;
}
// Optionally, provide a method to shut down the executor service
public void shutdown() {
executorService.shutdown();
}
}

20
openmetadata-service/src/main/java/org/openmetadata/service/util/CSVExportMessage.java Normal file
View File

@ -0,0 +1,20 @@
package org.openmetadata.service.util;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@NoArgsConstructor
public class CSVExportMessage {
@Getter @Setter private String jobId;
@Getter @Setter private String status;
@Getter @Setter private String data;
@Getter @Setter private String error;
public CSVExportMessage(String jobId, String status, String data, String error) {
this.jobId = jobId;
this.status = status;
this.data = data;
this.error = error;
}
}

16
openmetadata-service/src/main/java/org/openmetadata/service/util/CSVExportResponse.java Normal file
View File

@ -0,0 +1,16 @@
package org.openmetadata.service.util;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@NoArgsConstructor
public class CSVExportResponse {
@Getter @Setter private String jobId;
@Getter @Setter private String message;
public CSVExportResponse(String jobId, String message) {
this.jobId = jobId;
this.message = message;
}
}

25
openmetadata-service/src/main/java/org/openmetadata/service/util/WebsocketNotificationHandler.java
View File

@ -25,6 +25,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.ws.rs.container.ContainerResponseContext;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.entity.feed.Thread;
import org.openmetadata.schema.entity.teams.Team;
@ -57,6 +58,15 @@ public class WebsocketNotificationHandler {
});
}
public static void sendCsvExportCompleteNotification(
String jobId, SecurityContext securityContext, String csvData) {
CSVExportMessage message = new CSVExportMessage(jobId, "COMPLETED", csvData, null);
String jsonMessage = JsonUtils.pojoToJson(message);
UUID userId = getUserIdFromSecurityContext(securityContext);
WebSocketManager.getInstance()
.sendToOne(userId, WebSocketManager.CSV_EXPORT_CHANNEL, jsonMessage);
}
private void handleNotifications(ContainerResponseContext responseContext) {
int responseCode = responseContext.getStatus();
if (responseCode == Response.Status.CREATED.getStatusCode()
@ -149,4 +159,19 @@ public class WebsocketNotificationHandler {
}
});
}
public static void sendCsvExportFailedNotification(
String jobId, SecurityContext securityContext, String errorMessage) {
CSVExportMessage message = new CSVExportMessage(jobId, "FAILED", null, errorMessage);
String jsonMessage = JsonUtils.pojoToJson(message);
UUID userId = getUserIdFromSecurityContext(securityContext);
WebSocketManager.getInstance()
.sendToOne(userId, WebSocketManager.CSV_EXPORT_CHANNEL, jsonMessage);
}
private static UUID getUserIdFromSecurityContext(SecurityContext securityContext) {
String username = securityContext.getUserPrincipal().getName();
User user = Entity.getCollectionDAO().userDAO().findEntityByName(username);
return user.getId();
}
}

93
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -76,6 +76,8 @@ import es.org.elasticsearch.xcontent.NamedXContentRegistry;
import es.org.elasticsearch.xcontent.ParseField;
import es.org.elasticsearch.xcontent.XContentParser;
import es.org.elasticsearch.xcontent.json.JsonXContent;
import io.socket.client.IO;
import io.socket.client.Socket;
import java.io.IOException;
import java.net.URISyntaxException;
import java.time.Duration;
@ -91,6 +93,7 @@ import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
@ -214,6 +217,8 @@ import org.openmetadata.service.resources.tags.TagResourceTest;
import org.openmetadata.service.resources.teams.*;
import org.openmetadata.service.search.models.IndexMapping;
import org.openmetadata.service.security.SecurityUtil;
import org.openmetadata.service.util.CSVExportMessage;
import org.openmetadata.service.util.CSVExportResponse;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.FullyQualifiedName;
import org.openmetadata.service.util.JsonUtils;
@ -3967,9 +3972,91 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
return TestUtils.putCsv(target, csv, CsvImportResult.class, Status.OK, ADMIN_AUTH_HEADERS);
}
protected String exportCsv(String entityName) throws HttpResponseException {
private String receiveCsvViaSocketIO(String entityName) throws Exception {
UUID userId = getAdminUserId();
String uri = String.format("http://localhost:%d", APP.getLocalPort());
IO.Options options = new IO.Options();
options.path = "/api/v1/push/feed";
options.query = "userId=" + userId.toString();
options.transports = new String[] {"websocket"};
options.reconnection = false;
options.timeout = 10000; // 10 seconds
Socket socket = IO.socket(uri, options);
CountDownLatch connectLatch = new CountDownLatch(1);
CountDownLatch messageLatch = new CountDownLatch(1);
final String[] receivedMessage = new String[1];
socket
.on(
Socket.EVENT_CONNECT,
args -> {
System.out.println("Connected to Socket.IO server");
connectLatch.countDown();
})
.on(
"csvExportChannel",
args -> {
receivedMessage[0] = (String) args[0];
System.out.println("Received message: " + receivedMessage[0]);
messageLatch.countDown();
socket.disconnect();
})
.on(
Socket.EVENT_CONNECT_ERROR,
args -> {
System.err.println("Socket.IO connect error: " + args[0]);
connectLatch.countDown();
messageLatch.countDown();
})
.on(
Socket.EVENT_DISCONNECT,
args -> {
System.out.println("Disconnected from Socket.IO server");
});
socket.connect();
if (!connectLatch.await(10, TimeUnit.SECONDS)) {
fail("Could not connect to Socket.IO server");
}
// Initiate the export after connection is established
String jobId = initiateExport(entityName);
if (!messageLatch.await(30, TimeUnit.SECONDS)) {
fail("Did not receive CSV data via Socket.IO within the expected time.");
}
String receivedJson = receivedMessage[0];
if (receivedJson == null) {
fail("Received message is null.");
}
CSVExportMessage csvExportMessage = JsonUtils.readValue(receivedJson, CSVExportMessage.class);
if ("COMPLETED".equals(csvExportMessage.getStatus())) {
return csvExportMessage.getData();
} else if ("FAILED".equals(csvExportMessage.getStatus())) {
fail("CSV export failed: " + csvExportMessage.getError());
} else {
fail("Unknown status received: " + csvExportMessage.getStatus());
}
return null;
}
private UUID getAdminUserId() throws HttpResponseException {
UserResourceTest userResourceTest = new UserResourceTest();
User adminUser = userResourceTest.getEntityByName("admin", ADMIN_AUTH_HEADERS);
return adminUser.getId();
}
protected String initiateExport(String entityName) throws IOException {
WebTarget target = getResourceByName(entityName + "/export");
return TestUtils.get(target, String.class, ADMIN_AUTH_HEADERS);
CSVExportResponse response =
TestUtils.getWithResponse(
target, CSVExportResponse.class, ADMIN_AUTH_HEADERS, Status.ACCEPTED.getStatusCode());
return response.getJobId();
}
@SneakyThrows
@ -4001,7 +4088,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
assertEquals(dryRunResult.withDryRun(false), result);
// Finally, export CSV and ensure the exported CSV is same as imported CSV
String exportedCsv = exportCsv(entityName);
String exportedCsv = receiveCsvViaSocketIO(entityName);
CsvUtilTest.assertCsv(csv, exportedCsv);
}

8
openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java
View File

@ -139,6 +139,7 @@ import org.openmetadata.service.resources.databases.TableResourceTest;
import org.openmetadata.service.resources.teams.UserResource.UserList;
import org.openmetadata.service.security.AuthenticationException;
import org.openmetadata.service.security.mask.PIIMasker;
import org.openmetadata.service.util.CSVExportResponse;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.PasswordUtil;
@ -1560,9 +1561,12 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
}
@Override
protected String exportCsv(String teamName) throws HttpResponseException {
protected String initiateExport(String teamName) throws HttpResponseException {
WebTarget target = getCollection().path("/export");
target = target.queryParam("team", teamName);
return TestUtils.get(target, String.class, ADMIN_AUTH_HEADERS);
CSVExportResponse response =
TestUtils.getWithResponse(
target, CSVExportResponse.class, ADMIN_AUTH_HEADERS, Status.ACCEPTED.getStatusCode());
return response.getJobId();
}
}

5
openmetadata-ui/src/main/resources/ui/src/assets/svg/banner/ic-banner-error.svg Normal file
View File

@ -0,0 +1,5 @@
<svg width="25" height="26" viewBox="0 0 25 26" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M6.28843 0.5L0 6.78843V19.2116L6.28843 25.5H18.7116L25 19.2116V6.78843L18.7116 0.5L6.28843 0.5ZM23.5352 18.6048L18.1048 24.0352H6.89517L1.46484 18.6048V7.39517L6.89517 1.96484H18.1048L23.5352 7.39517V18.6048Z" fill="#FF4E27"/>
<path d="M12.5 14.928C11.9606 14.928 11.5234 14.4908 11.5234 13.9514V8.09204C11.5234 7.55269 11.9606 7.11548 12.5 7.11548C13.0394 7.11548 13.4766 7.55269 13.4766 8.09204V13.9514C13.4766 14.4908 13.0394 14.928 12.5 14.928Z" fill="#FF4E27"/>
<path d="M12.5 18.8342C13.0393 18.8342 13.4766 18.397 13.4766 17.8577C13.4766 17.3183 13.0393 16.8811 12.5 16.8811C11.9607 16.8811 11.5234 17.3183 11.5234 17.8577C11.5234 18.397 11.9607 18.8342 12.5 18.8342Z" fill="#FF4E27"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 805 B

10
openmetadata-ui/src/main/resources/ui/src/assets/svg/banner/ic-banner-success.svg Normal file
View File

@ -0,0 +1,10 @@
<svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_6713_202783)">
<path d="M18.0256 8.53367C18.4071 8.91514 18.4071 9.5335 18.0256 9.91478L11.4742 16.4663C11.0928 16.8476 10.4746 16.8476 10.0931 16.4663L6.97441 13.3474C6.59294 12.9662 6.59294 12.3478 6.97441 11.9665C7.35569 11.585 7.97405 11.585 8.35533 11.9665L10.7836 14.3948L16.6445 8.53367C17.0259 8.15239 17.6443 8.15239 18.0256 8.53367ZM25 12.5C25 19.4094 19.4084 25 12.5 25C5.59063 25 0 19.4084 0 12.5C0 5.59063 5.59158 0 12.5 0C19.4094 0 25 5.59158 25 12.5ZM23.0469 12.5C23.0469 6.67019 18.329 1.95312 12.5 1.95312C6.67019 1.95312 1.95312 6.67095 1.95312 12.5C1.95312 18.3298 6.67095 23.0469 12.5 23.0469C18.3298 23.0469 23.0469 18.329 23.0469 12.5Z" fill="#1D7C4D"/>
</g>
<defs>
<clipPath id="clip0_6713_202783">
<rect width="25" height="25" fill="white"/>
</clipPath>
</defs>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 914 B

87
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityExportModalProvider/EntityExportModalProvider.component.tsx
View File

@ -12,11 +12,15 @@
*/
import { Form, Input, Modal } from 'antd';
import { AxiosError } from 'axios';
import React, { ReactNode, useEffect, useMemo, useState } from 'react';
import { isString } from 'lodash';
import React, { ReactNode, useEffect, useMemo, useRef, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { getCurrentISODate } from '../../../utils/date-time/DateTimeUtils';
import { showErrorToast } from '../../../utils/ToastUtils';
import Banner, { BannerProps } from '../../common/Banner/Banner';
import {
CSVExportJob,
CSVExportWebsocketResponse,
EntityExportModalContextProps,
ExportData,
} from './EntityExportModalProvider.interface';
@ -35,6 +39,11 @@ export const EntityExportModalProvider = ({
const { t } = useTranslation();
const [exportData, setExportData] = useState<ExportData | null>(null);
const [downloading, setDownloading] = useState<boolean>(false);
const csvExportJobRef = useRef<Partial<CSVExportJob>>();
const [csvExportJob, setCSVExportJob] = useState<Partial<CSVExportJob>>();
const handleCancel = () => {
setExportData(null);
};
@ -70,11 +79,55 @@ export const EntityExportModalProvider = ({
setDownloading(true);
const data = await exportData.onExport(exportData.name);
handleDownload(data, fileName);
handleCancel();
if (isString(data)) {
handleDownload(data, fileName);
handleCancel();
setDownloading(false);
} else {
const jobData = {
jobId: data.jobId,
fileName: fileName,
message: data.message,
};
setCSVExportJob(jobData);
csvExportJobRef.current = jobData;
}
} catch (error) {
showErrorToast(error as AxiosError);
} finally {
setDownloading(false);
}
};
const handleCSVExportSuccess = (data: string, fileName?: string) => {
handleDownload(
data,
fileName ?? `${exportData?.name}_${getCurrentISODate()}`
);
setDownloading(false);
handleCancel();
setCSVExportJob(undefined);
csvExportJobRef.current = undefined;
};
const handleCSVExportJobUpdate = (
response: Partial<CSVExportWebsocketResponse>
) => {
const updatedCSVExportJob: Partial<CSVExportJob> = {
...response,
...csvExportJobRef.current,
};
setCSVExportJob(updatedCSVExportJob);
csvExportJobRef.current = updatedCSVExportJob;
if (response.status === 'COMPLETED' && response.data) {
handleCSVExportSuccess(
response.data ?? '',
csvExportJobRef.current?.fileName
);
} else {
setDownloading(false);
}
};
@ -88,7 +141,23 @@ export const EntityExportModalProvider = ({
}
}, [exportData]);
const providerValue = useMemo(() => ({ showModal }), []);
const providerValue = useMemo(
() => ({
showModal,
onUpdateCSVExportJob: handleCSVExportJobUpdate,
}),
[]
);
const bannerConfig = useMemo(() => {
const isCompleted = csvExportJob?.status === 'COMPLETED';
return {
type: isCompleted ? 'success' : 'error',
message: isCompleted ? csvExportJob?.message : csvExportJob?.error,
hasJobId: !!csvExportJob?.jobId,
};
}, [csvExportJob]);
return (
<EntityExportModalContext.Provider value={providerValue}>
@ -124,6 +193,14 @@ export const EntityExportModalProvider = ({
<Input addonAfter=".csv" data-testid="file-name-input" />
</Form.Item>
</Form>
{bannerConfig.hasJobId && bannerConfig.message && (
<Banner
className="border-radius"
message={bannerConfig.message}
type={bannerConfig.type as BannerProps['type']}
/>
)}
</Modal>
)}
</>

20
openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityExportModalProvider/EntityExportModalProvider.interface.ts
View File

@ -10,11 +10,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type CSVExportResponse = {
jobId: string;
message: string;
};
export type CSVExportWebsocketResponse = {
jobId: string;
status: 'COMPLETED' | 'FAILED';
data: string;
error: string | null;
};
export type CSVExportJob = {
fileName: string;
} & Partial<CSVExportWebsocketResponse> &
CSVExportResponse;
export type ExportData = {
name: string;
title?: string;
onExport: (name: string) => Promise<string>;
onExport: (name: string) => Promise<CSVExportResponse | string>;
};
export interface EntityExportModalContextProps {
showModal: (data: ExportData) => void;
onUpdateCSVExportJob: (data: Partial<CSVExportWebsocketResponse>) => void;
}

14
openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx
View File

@ -90,6 +90,8 @@ import { ActivityFeedTabs } from '../ActivityFeed/ActivityFeedTab/ActivityFeedTa
import SearchOptions from '../AppBar/SearchOptions';
import Suggestions from '../AppBar/Suggestions';
import CmdKIcon from '../common/CmdKIcon/CmdKIcon.component';
import { useEntityExportModalProvider } from '../Entity/EntityExportModalProvider/EntityExportModalProvider.component';
import { CSVExportWebsocketResponse } from '../Entity/EntityExportModalProvider/EntityExportModalProvider.interface';
import WhatsNewModal from '../Modals/WhatsNewModal/WhatsNewModal';
import NotificationBox from '../NotificationBox/NotificationBox.component';
import { UserProfileIcon } from '../Settings/Users/UserProfileIcon/UserProfileIcon.component';
@ -110,6 +112,7 @@ const NavBar = ({
handleOnClick,
handleClear,
}: NavBarProps) => {
const { onUpdateCSVExportJob } = useEntityExportModalProvider();
const { searchCriteria, updateSearchCriteria } = useApplicationStore();
const searchContainerRef = useRef<HTMLDivElement>(null);
const Logo = useMemo(() => brandImageClassBase.getMonogram().src, []);
@ -349,11 +352,22 @@ const NavBar = ({
);
}
});
socket.on(SOCKET_EVENTS.CSV_EXPORT_CHANNEL, (exportResponse) => {
if (exportResponse) {
const exportResponseData = JSON.parse(
exportResponse
) as CSVExportWebsocketResponse;
onUpdateCSVExportJob(exportResponseData);
}
});
}
return () => {
socket && socket.off(SOCKET_EVENTS.TASK_CHANNEL);
socket && socket.off(SOCKET_EVENTS.MENTION_CHANNEL);
socket && socket.off(SOCKET_EVENTS.CSV_EXPORT_CHANNEL);
};
}, [socket]);

22
openmetadata-ui/src/main/resources/ui/src/components/common/Banner/Banner.tsx Normal file
View File

@ -0,0 +1,22 @@
import React, { FC } from 'react';
import './banner.less';
import classNames from 'classnames';
import { ReactComponent as ErrorIcon } from '../../../assets/svg/banner/ic-banner-error.svg';
import { ReactComponent as SuccessIcon } from '../../../assets/svg/banner/ic-banner-success.svg';
export interface BannerProps extends React.HTMLAttributes<HTMLDivElement> {
type: 'success' | 'error';
message: string;
}
const Banner: FC<BannerProps> = ({ type, message, className }) => {
return (
<div className={classNames('message-banner-wrapper', type, className)}>
{type === 'success' ? <SuccessIcon /> : <ErrorIcon />}
<span className="message-banner-text">{message}</span>
</div>
);
};
export default Banner;

23
openmetadata-ui/src/main/resources/ui/src/components/common/Banner/banner.less Normal file
View File

@ -0,0 +1,23 @@
@import (reference) url('../../../styles/variables.less');
.message-banner-wrapper {
display: flex;
gap: 8px;
align-items: center;
backdrop-filter: blur(500px);
padding: 8px 16px;
&.border-radius {
border-radius: 4px;
}
&.success {
background-color: @success-bg-color;
color: @success-color;
}
&.error {
background-color: @error-bg-color;
color: @error-color;
}
}

1
openmetadata-ui/src/main/resources/ui/src/constants/constants.ts
View File

@ -295,6 +295,7 @@ export const SOCKET_EVENTS = {
TASK_CHANNEL: 'taskChannel',
MENTION_CHANNEL: 'mentionChannel',
JOB_STATUS: 'jobStatus',
CSV_EXPORT_CHANNEL: 'csvExportChannel',
};
export const IN_PAGE_SEARCH_ROUTES: Record<string, Array<string>> = {

1
openmetadata-ui/src/main/resources/ui/src/locale/languages/gl-es.json
View File

@ -773,6 +773,7 @@
"no-description": "Sen descrición",
"no-diff-available": "Non hai diferenzas dispoñibles",
"no-entity": "Non hai {{entity}}",
"no-entity-available": "No {{entity}} are available",
"no-entity-selected": "Non hai {{entity}} seleccionado",
"no-matching-data-asset": "Non se atoparon activos de datos coincidentes",
"no-of-test": "Nº de probas",

3
openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts
View File

@ -14,6 +14,7 @@
import { AxiosResponse } from 'axios';
import { Operation } from 'fast-json-patch';
import { PagingResponse } from 'Models';
import { CSVExportResponse } from '../components/Entity/EntityExportModalProvider/EntityExportModalProvider.interface';
import { VotingDataProps } from '../components/Entity/Voting/voting.interface';
import { ES_MAX_PAGE_SIZE, PAGE_SIZE_MEDIUM } from '../constants/constants';
import { TabSpecificField } from '../enums/entity.enum';
@ -172,7 +173,7 @@ export const deleteGlossaryTerm = (id: string) => {
};
export const exportGlossaryInCSVFormat = async (glossaryName: string) => {
const response = await APIClient.get<string>(
const response = await APIClient.get<CSVExportResponse>(
`/glossaries/name/${getEncodedFqn(glossaryName)}/export`
);

5
openmetadata-ui/src/main/resources/ui/src/rest/teamsAPI.ts
View File

@ -14,6 +14,7 @@
import { AxiosResponse } from 'axios';
import { Operation } from 'fast-json-patch';
import { PagingResponse, RestoreRequestType } from 'Models';
import { CSVExportResponse } from '../components/Entity/EntityExportModalProvider/EntityExportModalProvider.interface';
import { CreateTeam } from '../generated/api/teams/createTeam';
import { Team } from '../generated/entity/teams/team';
import { TeamHierarchy } from '../generated/entity/teams/teamHierarchy';
@ -90,7 +91,7 @@ export const restoreTeam = async (id: string) => {
};
export const exportTeam = async (teamName: string) => {
const response = await APIClient.get<string>(
const response = await APIClient.get<CSVExportResponse>(
`/teams/name/${getEncodedFqn(teamName)}/export`
);
@ -98,7 +99,7 @@ export const exportTeam = async (teamName: string) => {
};
export const exportUserOfTeam = async (team: string) => {
const response = await APIClient.get<string>(`/users/export`, {
const response = await APIClient.get<CSVExportResponse>(`/users/export`, {
params: { team },
});

5
openmetadata-ui/src/main/resources/ui/src/styles/variables.less
View File

@ -168,3 +168,8 @@
@margin-sm: 12px; // Form controls and items
@margin-xs: 8px; // small items
@margin-xss: 4px; // more small
@error-bg-color: rgb(from @error-color r g b / 0.1);
@success-bg-color: rgb(from @success-color r g b / 0.1);
@warning-bg-color: rgb(from @warning-color r g b / 0.1);
@info-bg-color: rgb(from @info-color r g b / 0.1);