diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java index c49cf38e1f6..2c374e939f0 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java @@ -147,7 +147,8 @@ public abstract class DashboardRepository { // Service can't be changed in update since service name is part of FQN and // change to a different service will result in a different FQN and creation of a new database under the new service storedDashboard.setService(service); - applyTags(updatedDashboard); + storedDashboard.setCharts(updatedDashboard.getCharts()); + updateChartRelationships(storedDashboard); return new PutResponse<>(Response.Status.OK, storedDashboard); } @@ -333,6 +334,17 @@ public abstract class DashboardRepository { applyTags(dashboard); } + private void updateChartRelationships(Dashboard dashboard) { + // Add relationship from dashboard to chart + String dashboardId = dashboard.getId().toString(); + if (dashboard.getCharts() != null) { + for (EntityReference chart : dashboard.getCharts()) { + relationshipDAO().insert(dashboardId, chart.getId().toString(), Entity.DASHBOARD, Entity.CHART, + Relationship.CONTAINS.ordinal()); + } + } + } + private Dashboard validateDashboard(String id) throws IOException { return EntityUtil.validate(id, dashboardDAO().findById(id), Dashboard.class); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java index 13d5ffe092d..e4c2e1a7da3 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java @@ -54,7 +54,7 @@ import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityN public abstract class TopicRepository { private static final Logger LOG = LoggerFactory.getLogger(TopicRepository.class); - private static final Fields TOPIC_UPDATE_FIELDS = new Fields(TopicResource.FIELD_LIST, "owner"); + private static final Fields TOPIC_UPDATE_FIELDS = new Fields(TopicResource.FIELD_LIST, "owner,tags"); private static final Fields TOPIC_PATCH_FIELDS = new Fields(TopicResource.FIELD_LIST, "owner,service,tags"); public static String getFQN(EntityReference service, Topic topic) { diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java index 4da9aa1a7b4..e991000abcd 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java @@ -23,14 +23,17 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.openmetadata.catalog.CatalogApplicationTest; import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.api.data.CreateChart; import org.openmetadata.catalog.api.data.CreateDashboard; import org.openmetadata.catalog.api.services.CreateDashboardService; import org.openmetadata.catalog.api.services.CreateDashboardService.DashboardServiceType; +import org.openmetadata.catalog.entity.data.Chart; import org.openmetadata.catalog.entity.data.Dashboard; import org.openmetadata.catalog.entity.services.DashboardService; import org.openmetadata.catalog.entity.teams.Team; import org.openmetadata.catalog.entity.teams.User; import org.openmetadata.catalog.exception.CatalogExceptionMessage; +import org.openmetadata.catalog.resources.charts.ChartResourceTest; import org.openmetadata.catalog.resources.dashboards.DashboardResource.DashboardList; import org.openmetadata.catalog.resources.services.DashboardServiceResourceTest; import org.openmetadata.catalog.resources.teams.TeamResourceTest; @@ -46,6 +49,8 @@ import org.slf4j.LoggerFactory; import javax.json.JsonPatch; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response.Status; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.UUID; @@ -59,6 +64,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound; import static org.openmetadata.catalog.exception.CatalogExceptionMessage.readOnlyAttribute; import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders; @@ -74,6 +80,7 @@ public class DashboardResourceTest extends CatalogApplicationTest { public static EntityReference TEAM_OWNER1; public static EntityReference SUPERSET_REFERENCE; public static EntityReference LOOKER_REFERENCE; + public static List CHART_REFERENCES; @BeforeAll public static void setup(TestInfo test) throws HttpResponseException { @@ -92,6 +99,12 @@ public class DashboardResourceTest extends CatalogApplicationTest { createService.withName("looker").withServiceType(DashboardServiceType.Looker); service = DashboardServiceResourceTest.createService(createService, adminAuthHeaders()); LOOKER_REFERENCE = EntityUtil.getEntityReference(service); + CHART_REFERENCES = new ArrayList<>(); + for (int i=0; i < 3; i++) { + CreateChart createChart = ChartResourceTest.create(test, i).withService(SUPERSET_REFERENCE); + Chart chart = ChartResourceTest.createChart(createChart, adminAuthHeaders()); + CHART_REFERENCES.add(EntityUtil.getEntityReference(chart)); + } } @@ -142,6 +155,11 @@ public class DashboardResourceTest extends CatalogApplicationTest { createAndCheckDashboard(create(test).withOwner(TEAM_OWNER1), adminAuthHeaders()); } + @Test + public void post_DashboardWithCharts_200_ok(TestInfo test) throws HttpResponseException { + createAndCheckDashboard(create(test), CHART_REFERENCES, adminAuthHeaders()); + } + @Test public void post_Dashboard_as_non_admin_401(TestInfo test) { CreateDashboard create = create(test); @@ -353,6 +371,19 @@ public class DashboardResourceTest extends CatalogApplicationTest { assertNull(db.getOwner()); } + + @Test + public void put_DashboardChartsUpdate_200(TestInfo test) throws HttpResponseException { + CreateDashboard request = create(test).withService(SUPERSET_REFERENCE).withDescription(null); + createAndCheckDashboard(request, adminAuthHeaders()); + + Dashboard dashboard = updateAndCheckDashboard(request + .withDescription("newDescription").withCharts(CHART_REFERENCES), + OK, adminAuthHeaders()); + validateDashboardCharts(dashboard, CHART_REFERENCES); + assertEquals("newDescription", dashboard.getDescription()); + } + @Test public void get_nonExistentDashboard_404_notFound() { HttpResponseException exception = assertThrows(HttpResponseException.class, () -> @@ -486,6 +517,14 @@ public class DashboardResourceTest extends CatalogApplicationTest { return getAndValidate(dashboard.getId(), create, authHeaders); } + public static Dashboard createAndCheckDashboard(CreateDashboard create, List charts, + Map authHeaders) throws HttpResponseException { + create.withCharts(charts); + Dashboard dashboard = createDashboard(create, authHeaders); + validateDashboard(dashboard, create.getDescription(), create.getOwner(), create.getService(), charts); + return getAndValidate(dashboard.getId(), create, authHeaders); + } + public static Dashboard updateAndCheckDashboard(CreateDashboard create, Status status, Map authHeaders) throws HttpResponseException { @@ -501,12 +540,12 @@ public class DashboardResourceTest extends CatalogApplicationTest { CreateDashboard create, Map authHeaders) throws HttpResponseException { // GET the newly created Dashboard by ID and validate - Dashboard dashboard = getDashboard(dashboardId, "service,owner", authHeaders); + Dashboard dashboard = getDashboard(dashboardId, "service,owner,charts", authHeaders); validateDashboard(dashboard, create.getDescription(), create.getOwner(), create.getService()); // GET the newly created Dashboard by name and validate String fqn = dashboard.getFullyQualifiedName(); - dashboard = getDashboardByName(fqn, "service,owner", authHeaders); + dashboard = getDashboardByName(fqn, "service,owner,charts", authHeaders); return validateDashboard(dashboard, create.getDescription(), create.getOwner(), create.getService()); } @@ -575,6 +614,46 @@ public class DashboardResourceTest extends CatalogApplicationTest { return dashboard; } + private static Dashboard validateDashboard(Dashboard dashboard, String expectedDescription, + EntityReference expectedOwner, EntityReference expectedService, + List charts) { + assertNotNull(dashboard.getId()); + assertNotNull(dashboard.getHref()); + assertEquals(expectedDescription, dashboard.getDescription()); + + // Validate owner + if (expectedOwner != null) { + TestUtils.validateEntityReference(dashboard.getOwner()); + assertEquals(expectedOwner.getId(), dashboard.getOwner().getId()); + assertEquals(expectedOwner.getType(), dashboard.getOwner().getType()); + assertNotNull(dashboard.getOwner().getHref()); + } + + // Validate service + if (expectedService != null) { + TestUtils.validateEntityReference(dashboard.getService()); + assertEquals(expectedService.getId(), dashboard.getService().getId()); + assertEquals(expectedService.getType(), dashboard.getService().getType()); + } + validateDashboardCharts(dashboard, charts); + return dashboard; + } + + private static void validateDashboardCharts(Dashboard dashboard, List charts) { + if (charts != null) { + List expectedChartReferences = new ArrayList<>(); + for (EntityReference chart: charts) { + expectedChartReferences.add(chart.getId()); + } + List actualChartReferences = new ArrayList<>(); + for (EntityReference chart: dashboard.getCharts()) { + TestUtils.validateEntityReference(chart); + actualChartReferences.add(chart.getId()); + } + assertTrue(actualChartReferences.containsAll(expectedChartReferences)); + } + } + private Dashboard patchDashboardAttributesAndCheck(Dashboard dashboard, String newDescription, EntityReference newOwner, Map authHeaders) throws JsonProcessingException, HttpResponseException { diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/DatabaseResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/DatabaseResourceTest.java index abdc88e5373..134933af694 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/DatabaseResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/databases/DatabaseResourceTest.java @@ -26,11 +26,13 @@ import org.openmetadata.catalog.Entity; import org.openmetadata.catalog.api.data.CreateDatabase; import org.openmetadata.catalog.api.services.CreateDatabaseService; import org.openmetadata.catalog.api.services.CreateDatabaseService.DatabaseServiceType; +import org.openmetadata.catalog.entity.data.Chart; import org.openmetadata.catalog.entity.data.Database; import org.openmetadata.catalog.entity.services.DatabaseService; import org.openmetadata.catalog.entity.teams.Team; import org.openmetadata.catalog.entity.teams.User; import org.openmetadata.catalog.exception.CatalogExceptionMessage; +import org.openmetadata.catalog.resources.charts.ChartResourceTest; import org.openmetadata.catalog.resources.databases.DatabaseResource.DatabaseList; import org.openmetadata.catalog.resources.services.DatabaseServiceResourceTest; import org.openmetadata.catalog.resources.teams.TeamResourceTest; @@ -46,6 +48,7 @@ import org.slf4j.LoggerFactory; import javax.json.JsonPatch; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response.Status; +import java.util.List; import java.util.Map; import java.util.UUID; @@ -101,6 +104,7 @@ public class DatabaseResourceTest extends CatalogApplicationTest { createService.withName("mysqlDB").withServiceType(DatabaseServiceType.MySQL); service = DatabaseServiceResourceTest.createService(createService, adminAuthHeaders()); MYSQL_REFERENCE = EntityUtil.getEntityReference(service); + } @Test diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/MessagingServiceResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/MessagingServiceResourceTest.java index 8235d3e3985..5e339771c43 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/MessagingServiceResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/MessagingServiceResourceTest.java @@ -17,6 +17,7 @@ package org.openmetadata.catalog.resources.services; import org.apache.http.client.HttpResponseException; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.openmetadata.catalog.CatalogApplicationTest; @@ -26,16 +27,26 @@ import org.openmetadata.catalog.api.services.CreateMessagingService.MessagingSer import org.openmetadata.catalog.api.services.UpdateMessagingService; import org.openmetadata.catalog.entity.services.MessagingService; import org.openmetadata.catalog.exception.CatalogExceptionMessage; +import org.openmetadata.catalog.resources.tags.TagResourceTest; +import org.openmetadata.catalog.resources.teams.TeamResourceTest; +import org.openmetadata.catalog.resources.teams.UserResourceTest; +import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.Schedule; +import org.openmetadata.catalog.type.TagLabel; +import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.RestUtil; import org.openmetadata.catalog.util.TestUtils; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response.Status; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import static javax.ws.rs.core.Response.Status.CONFLICT; @@ -45,10 +56,21 @@ import static javax.ws.rs.core.Response.Status.OK; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders; import static org.openmetadata.catalog.util.TestUtils.authHeaders; public class MessagingServiceResourceTest extends CatalogApplicationTest { + + public static List KAFKA_BROKERS; + public static URI SCHEMA_REGISTRY_URL; + + @BeforeAll + public static void setup(TestInfo test) throws URISyntaxException { + KAFKA_BROKERS = List.of("192.168.1.1:0"); + SCHEMA_REGISTRY_URL = new URI("http://localhost:0"); + } + @Test public void post_serviceWithLongName_400_badRequest(TestInfo test) { // Create messaging with mandatory name field empty @@ -183,9 +205,9 @@ public class MessagingServiceResourceTest extends CatalogApplicationTest { } @Test - public void put_updateService_as_admin_2xx(TestInfo test) throws HttpResponseException { - MessagingService dbService = createAndCheckService(create(test).withDescription(null).withIngestionSchedule(null), - adminAuthHeaders()); + public void put_updateService_as_admin_2xx(TestInfo test) throws HttpResponseException, URISyntaxException { + MessagingService dbService = createAndCheckService(create(test).withDescription(null).withIngestionSchedule(null) + .withBrokers(KAFKA_BROKERS).withSchemaRegistry(SCHEMA_REGISTRY_URL), adminAuthHeaders()); String id = dbService.getId().toString(); // Update messaging description and ingestion service that are null @@ -200,6 +222,11 @@ public class MessagingServiceResourceTest extends CatalogApplicationTest { update.withDescription("description1").withIngestionSchedule(schedule.withRepeatFrequency("PT1H")); updateAndCheckService(id, update, OK, adminAuthHeaders()); + // update broker list and schema registry + update.withBrokers(List.of("localhost:0")).withSchemaRegistry(new URI("http://localhost:9000")); + updateAndCheckService(id, update, OK, adminAuthHeaders()); + MessagingService updatedService = getService(dbService.getId(), adminAuthHeaders()); + validateMessagingServiceConfig(updatedService, List.of("localhost:0"), new URI("http://localhost:9000")); } @Test @@ -337,7 +364,8 @@ public class MessagingServiceResourceTest extends CatalogApplicationTest { public static CreateMessagingService create(TestInfo test) { return new CreateMessagingService().withName(getName(test)).withServiceType(MessagingServiceType.Kafka) - .withBrokers(List.of("192.1.1.1:0")) + .withBrokers(KAFKA_BROKERS) + .withSchemaRegistry(SCHEMA_REGISTRY_URL) .withIngestionSchedule(new Schedule().withStartDate(new Date()).withRepeatFrequency("P1D")); } @@ -367,4 +395,11 @@ public class MessagingServiceResourceTest extends CatalogApplicationTest { return TestUtils.put(CatalogApplicationTest.getResource("services/messagingServices/" + id), updated, MessagingService.class, status, authHeaders); } + + private static void validateMessagingServiceConfig(MessagingService actualService, List expectedBrokers, + URI expectedSchemaRegistry) + throws HttpResponseException { + assertTrue(actualService.getBrokers().containsAll(expectedBrokers)); + assertEquals(actualService.getSchemaRegistry(), expectedSchemaRegistry); + } }