From b3b5f44c23c6f47c6a38a2008aa2596d77a036a2 Mon Sep 17 00:00:00 2001 From: Alberto Miorin <32617+amiorin@users.noreply.github.com> Date: Thu, 17 Feb 2022 08:21:53 +0100 Subject: [PATCH] Fix #1876: Allow dots in entity names (#2700) Add getter and setter for `name` into EntityInterface Refactor code in utility method. Rename the escape method. Fix webhook. Fix pipeline tests Fix airflowPipeline tests Fix role, team, and user tests Replace . with _DOT_ to keep compatibility with the past and avoid a migration. Remove the Python code that was replacing the . with _DOT_. Restore "no dots" for services and implement the JSON Schema rule. ** Entites escaping dots 1. AirflowPipeline 2. Chart 3. Dashboard 4. Database 5. Location 6. Metrics 7. MlModel 8. Pipeline 9. Table 10. Topic ** Entities rejecting dots 1. Database Service 2. Dashboard Service 3. Messaging Service 4. Pipeline Service 5. Storage Service ** Entities accepting dots 1. Bots 2. Policy 3. Report 4. Role 5. Team 6. User 7. Webhook ** Fields escaping dots 1. Column 2. Task Add escape for Column and TableConstraints Add escape to Task Remove the Python code. It will be added in a follow-up PR. Fix Pipeline entity tests --- .../jdbi3/AirflowPipelineRepository.java | 11 +++++ .../catalog/jdbi3/BotsRepository.java | 14 ++++-- .../catalog/jdbi3/ChartRepository.java | 11 +++++ .../catalog/jdbi3/DashboardRepository.java | 11 +++++ .../jdbi3/DashboardServiceRepository.java | 10 +++++ .../catalog/jdbi3/DatabaseRepository.java | 11 +++++ .../jdbi3/DatabaseServiceRepository.java | 10 +++++ .../catalog/jdbi3/GlossaryRepository.java | 10 +++++ .../catalog/jdbi3/GlossaryTermRepository.java | 10 +++++ .../catalog/jdbi3/LocationRepository.java | 11 +++++ .../jdbi3/MessagingServiceRepository.java | 10 +++++ .../catalog/jdbi3/MetricsRepository.java | 11 +++++ .../catalog/jdbi3/MlModelRepository.java | 11 +++++ .../catalog/jdbi3/PipelineRepository.java | 12 ++++++ .../jdbi3/PipelineServiceRepository.java | 10 +++++ .../catalog/jdbi3/PolicyRepository.java | 10 +++++ .../catalog/jdbi3/ReportRepository.java | 10 +++++ .../catalog/jdbi3/RoleRepository.java | 10 +++++ .../jdbi3/StorageServiceRepository.java | 10 +++++ .../catalog/jdbi3/TableRepository.java | 14 ++++++ .../catalog/jdbi3/TeamRepository.java | 10 +++++ .../catalog/jdbi3/TopicRepository.java | 11 +++++ .../catalog/jdbi3/UserRepository.java | 10 +++++ .../catalog/jdbi3/WebhookRepository.java | 16 +++++-- .../catalog/util/EntityInterface.java | 4 ++ .../openmetadata/catalog/util/EntityUtil.java | 28 ++++++++++++ .../api/services/createDashboardService.json | 3 +- .../api/services/createDatabaseService.json | 3 +- .../api/services/createMessagingService.json | 3 +- .../api/services/createPipelineService.json | 3 +- .../api/services/createStorageService.json | 3 +- .../json/schema/entity/data/database.json | 3 +- .../json/schema/entity/data/table.json | 12 +++--- .../json/schema/entity/data/topic.json | 3 +- .../json/schema/entity/events/webhook.json | 4 ++ .../entity/services/dashboardService.json | 3 +- .../entity/services/databaseService.json | 3 +- .../entity/services/messagingService.json | 3 +- .../entity/services/pipelineService.json | 3 +- .../entity/services/storageService.json | 3 +- .../EntityOperationsResourceTest.java | 6 ++- .../catalog/resources/EntityResourceTest.java | 39 +++++++++++++++-- .../resources/charts/ChartResourceTest.java | 2 +- .../dashboards/DashboardResourceTest.java | 1 + .../databases/DatabaseResourceTest.java | 1 + .../databases/TableResourceTest.java | 43 +++++++++++++++++-- .../resources/events/WebhookResourceTest.java | 2 +- .../glossary/GlossaryResourceTest.java | 3 +- .../glossary/GlossaryTermResourceTest.java | 1 + .../locations/LocationResourceTest.java | 1 + .../mlmodels/MlModelResourceTest.java | 12 +++++- .../AirflowPipelineResourceTest.java | 2 +- .../pipelines/PipelineResourceTest.java | 36 ++++++++++++++-- .../policies/PolicyResourceTest.java | 12 +++++- .../DashboardServiceResourceTest.java | 1 + .../services/DatabaseServiceResourceTest.java | 1 + .../MessagingServiceResourceTest.java | 1 + .../services/PipelineServiceResourceTest.java | 1 + .../services/StorageServiceResourceTest.java | 1 + .../resources/teams/RoleResourceTest.java | 4 +- .../resources/teams/TeamResourceTest.java | 3 +- .../resources/teams/UserResourceTest.java | 2 +- .../resources/topics/TopicResourceTest.java | 2 +- ingestion-core/src/metadata/_version.py | 2 +- 64 files changed, 474 insertions(+), 53 deletions(-) diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/AirflowPipelineRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/AirflowPipelineRepository.java index c2467f55cb5..b037382dc91 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/AirflowPipelineRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/AirflowPipelineRepository.java @@ -84,6 +84,7 @@ public class AirflowPipelineRepository extends EntityRepository @Override public void prepare(AirflowPipeline airflowPipeline) throws IOException, ParseException { + EntityUtil.escapeReservedChars(getEntityInterface(airflowPipeline)); EntityReference entityReference = helper(helper(airflowPipeline).findEntity("service", List.of(DATABASE_SERVICE, DASHBOARD_SERVICE))) .toEntityReference(); @@ -147,6 +148,11 @@ public class AirflowPipelineRepository extends EntityRepository return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -224,6 +230,11 @@ public class AirflowPipelineRepository extends EntityRepository entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java index c4061198864..9e6f0753e1e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/BotsRepository.java @@ -55,9 +55,7 @@ public class BotsRepository extends EntityRepository { } @Override - public void prepare(Bots entity) { - /* Nothing to do */ - } + public void prepare(Bots entity) {} @Override public void storeEntity(Bots entity, boolean update) throws IOException { @@ -91,6 +89,11 @@ public class BotsRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -161,6 +164,11 @@ public class BotsRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java index 7ff34666c14..5d65dc52217 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ChartRepository.java @@ -60,6 +60,7 @@ public class ChartRepository extends EntityRepository { @Override public void prepare(Chart chart) throws IOException, ParseException { + EntityUtil.escapeReservedChars(getEntityInterface(chart)); DashboardService dashboardService = helper(chart).findEntity("service", DASHBOARD_SERVICE); chart.setService(helper(dashboardService).toEntityReference()); chart.setServiceType(dashboardService.getServiceType()); @@ -142,6 +143,11 @@ public class ChartRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -222,6 +228,11 @@ public class ChartRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java index a0ee9bcd392..baf82a2ddca 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardRepository.java @@ -128,6 +128,7 @@ public class DashboardRepository extends EntityRepository { @Override public void prepare(Dashboard dashboard) throws IOException { + EntityUtil.escapeReservedChars(getEntityInterface(dashboard)); populateService(dashboard); dashboard.setFullyQualifiedName(getFQN(dashboard)); EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), dashboard.getOwner()); // Validate owner @@ -243,6 +244,11 @@ public class DashboardRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -326,6 +332,11 @@ public class DashboardRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java index 2f45cf7e86f..49b738b1f1a 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DashboardServiceRepository.java @@ -115,6 +115,11 @@ public class DashboardServiceRepository extends EntityRepository { @Override public void prepare(Database database) throws IOException { + EntityUtil.escapeReservedChars(getEntityInterface(database)); populateService(database); database.setFullyQualifiedName(getFQN(database)); database.setOwner( @@ -210,6 +211,11 @@ public class DatabaseRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -282,6 +288,11 @@ public class DatabaseRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java index 3de5bb2b54e..e882ec1972f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/DatabaseServiceRepository.java @@ -174,6 +174,11 @@ public class DatabaseServiceRepository extends EntityRepository return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public EntityReference getOwner() { return entity.getOwner(); @@ -244,6 +249,11 @@ public class DatabaseServiceRepository extends EntityRepository entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java index 4cfa70f52ad..791cb02ad96 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryRepository.java @@ -136,6 +136,11 @@ public class GlossaryRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -216,6 +221,11 @@ public class GlossaryRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java index f4f176d3c6b..ad7af57be9c 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/GlossaryTermRepository.java @@ -252,6 +252,11 @@ public class GlossaryTermRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -332,6 +337,11 @@ public class GlossaryTermRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java index 2a289e79c95..ff78334c696 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/LocationRepository.java @@ -177,6 +177,7 @@ public class LocationRepository extends EntityRepository { @Override public void prepare(Location location) throws IOException { + EntityUtil.escapeReservedChars(getEntityInterface(location)); StorageService storageService = getService(location.getService().getId(), location.getService().getType()); location.setService( new StorageServiceRepository.StorageServiceEntityInterface(storageService).getEntityReference()); @@ -260,6 +261,11 @@ public class LocationRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -347,6 +353,11 @@ public class LocationRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java index ac088ff3917..29cfcf5fcee 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MessagingServiceRepository.java @@ -119,6 +119,11 @@ public class MessagingServiceRepository extends EntityRepository { @Override public void prepare(Metrics metrics) throws IOException { + EntityUtil.escapeReservedChars(getEntityInterface(metrics)); metrics.setFullyQualifiedName(getFQN(metrics)); EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), metrics.getOwner()); // Validate owner metrics.setService(getService(metrics.getService())); @@ -139,6 +140,11 @@ public class MetricsRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -219,6 +225,11 @@ public class MetricsRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java index 3c44f4b75ee..2c8fdeab9c3 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java @@ -147,6 +147,7 @@ public class MlModelRepository extends EntityRepository { @Override public void prepare(MlModel mlModel) throws IOException { + EntityUtil.escapeReservedChars(getEntityInterface(mlModel)); mlModel.setFullyQualifiedName(getFQN(mlModel)); if (mlModel.getMlFeatures() != null && !mlModel.getMlFeatures().isEmpty()) { @@ -250,6 +251,11 @@ public class MlModelRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -335,6 +341,11 @@ public class MlModelRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java index 19b7016bf26..3557f1a7534 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineRepository.java @@ -163,6 +163,8 @@ public class PipelineRepository extends EntityRepository { @Override public void prepare(Pipeline pipeline) throws IOException { + EntityUtil.escapeReservedChars(getEntityInterface(pipeline)); + EntityUtil.escapeReservedChars(pipeline.getTasks()); populateService(pipeline); pipeline.setFullyQualifiedName(getFQN(pipeline)); EntityUtil.populateOwner(daoCollection.userDAO(), daoCollection.teamDAO(), pipeline.getOwner()); // Validate owner @@ -242,6 +244,11 @@ public class PipelineRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -324,6 +331,11 @@ public class PipelineRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java index d9ddddb266e..5523ffff955 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PipelineServiceRepository.java @@ -115,6 +115,11 @@ public class PipelineServiceRepository extends EntityRepository return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -185,6 +190,11 @@ public class PipelineServiceRepository extends EntityRepository entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java index 0e68ed72d87..ac31c5af852 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java @@ -285,6 +285,11 @@ public class PolicyRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -354,6 +359,11 @@ public class PolicyRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java index 893bb6a6ea9..3747f91ee65 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/ReportRepository.java @@ -112,6 +112,11 @@ public class ReportRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -192,6 +197,11 @@ public class ReportRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/RoleRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/RoleRepository.java index d0151cda116..8a99e0d3d54 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/RoleRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/RoleRepository.java @@ -201,6 +201,11 @@ public class RoleRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -262,6 +267,11 @@ public class RoleRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java index 7818cbb2f82..1e871f31988 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/StorageServiceRepository.java @@ -108,6 +108,11 @@ public class StorageServiceRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -178,6 +183,11 @@ public class StorageServiceRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java index 1e8be1898a9..ad2e72f3e39 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TableRepository.java @@ -302,6 +302,9 @@ public class TableRepository extends EntityRepository { @Override public void prepare(Table table) throws IOException, ParseException { + EntityUtil.escapeReservedChars(getEntityInterface(table)); + EntityUtil.escapeReservedChars(table.getColumns()); + EntityUtil.escapeReservedChars(table.getTableConstraints()); Database database = helper(table).findEntity("database", DATABASE); table.setDatabase(helper(database).toEntityReference()); DatabaseService databaseService = helper(database).findEntity("service", DATABASE_SERVICE); @@ -391,6 +394,7 @@ public class TableRepository extends EntityRepository
{ return new Column() .withDescription(column.getDescription()) .withName(column.getName()) + .withDisplayName(column.getDisplayName()) .withFullyQualifiedName(column.getFullyQualifiedName()) .withArrayDataType(column.getArrayDataType()) .withConstraint(column.getConstraint()) @@ -659,6 +663,11 @@ public class TableRepository extends EntityRepository
{ return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -744,6 +753,11 @@ public class TableRepository extends EntityRepository
{ entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java index bd8acc2ede2..fa14864a48c 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TeamRepository.java @@ -164,6 +164,11 @@ public class TeamRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -225,6 +230,11 @@ public class TeamRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java index 30a1e93af3d..36c0b863f50 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/TopicRepository.java @@ -68,6 +68,7 @@ public class TopicRepository extends EntityRepository { @Override public void prepare(Topic topic) throws IOException, ParseException { + EntityUtil.escapeReservedChars(getEntityInterface(topic)); MessagingService messagingService = helper(topic).findEntity("service", MESSAGING_SERVICE); topic.setService(helper(messagingService).toEntityReference()); topic.setServiceType(messagingService.getServiceType()); @@ -156,6 +157,11 @@ public class TopicRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -236,6 +242,11 @@ public class TopicRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java index a883ebdf0ae..32724b83f67 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/UserRepository.java @@ -219,6 +219,11 @@ public class UserRepository extends EntityRepository { return entity.getDisplayName(); } + @Override + public String getName() { + return entity.getName(); + } + @Override public Boolean isDeleted() { return entity.getDeleted(); @@ -280,6 +285,11 @@ public class UserRepository extends EntityRepository { entity.setDisplayName(displayName); } + @Override + public void setName(String name) { + entity.setName(name); + } + @Override public void setUpdateDetails(String updatedBy, long updatedAt) { entity.setUpdatedBy(updatedBy); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/WebhookRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/WebhookRepository.java index 635b03f9cc9..ad1515dbc28 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/WebhookRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/WebhookRepository.java @@ -84,9 +84,7 @@ public class WebhookRepository extends EntityRepository { } @Override - public void prepare(Webhook entity) throws IOException { - // Nothing to prepare - } + public void prepare(Webhook entity) throws IOException {} @Override public void storeEntity(Webhook entity, boolean update) throws IOException { @@ -185,6 +183,11 @@ public class WebhookRepository extends EntityRepository { @Override public String getDisplayName() { + return entity.getDisplayName(); + } + + @Override + public String getName() { return entity.getName(); } @@ -250,7 +253,12 @@ public class WebhookRepository extends EntityRepository { @Override public void setDisplayName(String displayName) { - /* No display name */ + entity.setDisplayName(displayName); + } + + @Override + public void setName(String name) { + entity.setName(name); } @Override diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityInterface.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityInterface.java index f293d24f4ba..6f695dfc584 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityInterface.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityInterface.java @@ -28,6 +28,8 @@ public interface EntityInterface { String getDisplayName(); + String getName(); + Boolean isDeleted(); default EntityReference getOwner() { @@ -72,6 +74,8 @@ public interface EntityInterface { void setDisplayName(String displayName); + void setName(String name); + void setUpdateDetails(String updatedBy, long updatedAt); void setChangeDescription(Double newVersion, ChangeDescription changeDescription); diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java index 9a16d782de2..6de48fccac7 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java @@ -476,4 +476,32 @@ public final class EntityUtil { .withEventType(EventType.ENTITY_SOFT_DELETED) .withEntities(eventFilter.getEntities()))); } + + public static void escapeReservedChars(EntityInterface entityInterface) { + entityInterface.setDisplayName( + entityInterface.getDisplayName() != null ? entityInterface.getDisplayName() : entityInterface.getName()); + entityInterface.setName(entityInterface.getName().replace(".", "_DOT_")); + } + + public static void escapeReservedChars(List collection) { + if (collection == null || collection.isEmpty()) { + return; + } + for (Object object : collection) { + if (object instanceof Column) { + Column column = (Column) object; + column.setDisplayName(column.getDisplayName() != null ? column.getDisplayName() : column.getName()); + column.setName(column.getName().replace(".", "_DOT_")); + escapeReservedChars(column.getChildren()); + } else if (object instanceof TableConstraint) { + TableConstraint constraint = (TableConstraint) object; + constraint.setColumns( + constraint.getColumns().stream().map(s -> s.replace(".", "_DOT_")).collect(Collectors.toList())); + } else if (object instanceof Task) { + Task task = (Task) object; + task.setDisplayName(task.getDisplayName() != null ? task.getDisplayName() : task.getName()); + task.setName(task.getName().replace(".", "_DOT_")); + } + } + } } diff --git a/catalog-rest-service/src/main/resources/json/schema/api/services/createDashboardService.json b/catalog-rest-service/src/main/resources/json/schema/api/services/createDashboardService.json index 9c72e6443be..abdb9ca9a74 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/services/createDashboardService.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/services/createDashboardService.json @@ -9,7 +9,8 @@ "description": "Name that identifies the this entity instance uniquely", "type": "string", "minLength": 1, - "maxLength": 128 + "maxLength": 128, + "pattern": "^[^.]*$" }, "description": { "description": "Description of dashboard service entity.", diff --git a/catalog-rest-service/src/main/resources/json/schema/api/services/createDatabaseService.json b/catalog-rest-service/src/main/resources/json/schema/api/services/createDatabaseService.json index 65796b739cf..6903f0452cf 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/services/createDatabaseService.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/services/createDatabaseService.json @@ -10,7 +10,8 @@ "description": "Name that identifies the this entity instance uniquely", "type": "string", "minLength": 1, - "maxLength": 128 + "maxLength": 128, + "pattern": "^[^.]*$" }, "description": { "description": "Description of Database entity.", diff --git a/catalog-rest-service/src/main/resources/json/schema/api/services/createMessagingService.json b/catalog-rest-service/src/main/resources/json/schema/api/services/createMessagingService.json index 7fc84948c4e..7d1e1da5c44 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/services/createMessagingService.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/services/createMessagingService.json @@ -9,7 +9,8 @@ "description": "Name that identifies the this entity instance uniquely", "type": "string", "minLength": 1, - "maxLength": 128 + "maxLength": 128, + "pattern": "^[^.]*$" }, "description": { "description": "Description of messaging service entity.", diff --git a/catalog-rest-service/src/main/resources/json/schema/api/services/createPipelineService.json b/catalog-rest-service/src/main/resources/json/schema/api/services/createPipelineService.json index 01cf788fee5..58491d886eb 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/services/createPipelineService.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/services/createPipelineService.json @@ -9,7 +9,8 @@ "description": "Name that identifies the this entity instance uniquely", "type": "string", "minLength": 1, - "maxLength": 128 + "maxLength": 128, + "pattern": "^[^.]*$" }, "description": { "description": "Description of pipeline service entity.", diff --git a/catalog-rest-service/src/main/resources/json/schema/api/services/createStorageService.json b/catalog-rest-service/src/main/resources/json/schema/api/services/createStorageService.json index 5195d88f81f..87a73a55a36 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/services/createStorageService.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/services/createStorageService.json @@ -9,7 +9,8 @@ "description": "Name that identifies the this entity instance uniquely", "type": "string", "minLength": 1, - "maxLength": 128 + "maxLength": 128, + "pattern": "^[^.]*$" }, "description": { "description": "Description of Storage entity.", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/database.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/database.json index a24270944c7..2aaffa4dc0a 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/data/database.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/database.json @@ -10,8 +10,7 @@ "description": "Name that identifies the database.", "type": "string", "minLength": 1, - "maxLength": 128, - "pattern": "^[^.]*$" + "maxLength": 128 } }, "properties": { diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json index 21403669edd..f3ebccc9b19 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/table.json @@ -103,15 +103,13 @@ "description": "Local name (not fully qualified name) of the column. ColumnName is `-` when the column is not named in struct dataType. For example, BigQuery supports struct with unnamed fields.", "type": "string", "minLength": 1, - "maxLength": 128, - "pattern": "^[^.]*$" + "maxLength": 128 }, "tableName": { - "description": "Local name (not fully qualified name) of a table.", + "description": "Local name (not fully qualified name) of a table. Dots will be escaped automatically", "type": "string", "minLength": 1, - "maxLength": 128, - "pattern": "^[^.]*$" + "maxLength": 128 }, "fullyQualifiedColumnName": { "description": "Fully qualified name of the column that includes `serviceName.databaseName.tableName.columnName[.nestedColumnName]`. When columnName is null for dataType struct fields, `field_#` where `#` is field index is used. For map dataType, for key the field name `key` is used and for the value field `value` is used.", @@ -127,6 +125,10 @@ "name": { "$ref": "#/definitions/columnName" }, + "displayName": { + "description": "Display Name that identifies this column name.", + "type": "string" + }, "dataType": { "description": "Data type of the column (int, date etc.).", "$ref": "#/definitions/dataType" diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/data/topic.json b/catalog-rest-service/src/main/resources/json/schema/entity/data/topic.json index 4cf6fae4e58..bb689b13793 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/data/topic.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/data/topic.json @@ -10,8 +10,7 @@ "type": "string", "javaType": "org.openmetadata.catalog.type.topic.TopicName", "minLength": 1, - "maxLength": 128, - "pattern": "^[^.]*$" + "maxLength": 128 }, "schemaType": { "description": "Schema type used for the message.", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json b/catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json index 9336aaf4bc8..9895807f959 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/events/webhook.json @@ -16,6 +16,10 @@ "minLength": 1, "maxLength": 128 }, + "displayName": { + "description": "Display Name that identifies this webhook.", + "type": "string" + }, "description": { "description": "Description of the application.", "type": "string" diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json index 9c5cfed726c..78900832b90 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/dashboardService.json @@ -37,7 +37,8 @@ "description": "Name that identifies this dashboard service.", "type": "string", "minLength": 1, - "maxLength": 128 + "maxLength": 128, + "pattern": "^[^.]*$" }, "displayName": { "description": "Display Name that identifies this dashboard service.", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json index 1b4df17eade..81502b9aa59 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/databaseService.json @@ -139,7 +139,8 @@ "description": "Name that identifies this database service.", "type": "string", "minLength": 1, - "maxLength": 128 + "maxLength": 128, + "pattern": "^[^.]*$" }, "displayName": { "description": "Display Name that identifies this database service.", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json index 2c72aedb546..5c17c65b3ea 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/messagingService.json @@ -36,7 +36,8 @@ "description": "Name that identifies this messaging service.", "type": "string", "minLength": 1, - "maxLength": 128 + "maxLength": 128, + "pattern": "^[^.]*$" }, "serviceType": { "description": "Type of messaging service such as Kafka or Pulsar...", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json index dffdd678cf9..80faa472fa3 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/pipelineService.json @@ -31,7 +31,8 @@ "description": "Name that identifies this pipeline service.", "type": "string", "minLength": 1, - "maxLength": 128 + "maxLength": 128, + "pattern": "^[^.]*$" }, "serviceType": { "description": "Type of pipeline service such as Airflow or Prefect...", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/storageService.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/storageService.json index fb350ba13f8..a1516310868 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/storageService.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/storageService.json @@ -14,7 +14,8 @@ "description": "Name that identifies this storage service.", "type": "string", "minLength": 1, - "maxLength": 128 + "maxLength": 128, + "pattern": "^[^.]*$" }, "displayName": { "description": "Display Name that identifies this storage service.", diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityOperationsResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityOperationsResourceTest.java index 1f7e580d33f..316cc1e7aa7 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityOperationsResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityOperationsResourceTest.java @@ -28,7 +28,8 @@ public abstract class EntityOperationsResourceTest extends EntityResourceT boolean supportsFollowers, boolean supportsOwner, boolean supportsTags, - boolean supportsAuthorizedMetadataOperations) { + boolean supportsAuthorizedMetadataOperations, + boolean supportsDots) { super( entityType, entityClass, @@ -38,7 +39,8 @@ public abstract class EntityOperationsResourceTest extends EntityResourceT supportsFollowers, supportsOwner, supportsTags, - supportsAuthorizedMetadataOperations); + supportsAuthorizedMetadataOperations, + supportsDots); } // Override the resource path name of regular entities api/v1/ to api/operations/v1/ diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java index 7f5491766ce..df12ceb0555 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/EntityResourceTest.java @@ -147,6 +147,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { private final boolean supportsFollowers; private final boolean supportsOwner; private final boolean supportsTags; + private final boolean supportsDots; protected boolean supportsPatch = true; protected boolean supportsSoftDelete = true; private final boolean supportsAuthorizedMetadataOperations; @@ -205,7 +206,9 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { boolean supportsFollowers, boolean supportsOwner, boolean supportsTags, - boolean supportsAuthorizedMetadataOperations) { + boolean supportsAuthorizedMetadataOperations, + boolean supportsDots) { + this.entityType = entityTYpe; this.entityClass = entityClass; this.entityListClass = entityListClass; @@ -215,6 +218,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { this.supportsOwner = supportsOwner; this.supportsTags = supportsTags; this.supportsAuthorizedMetadataOperations = supportsAuthorizedMetadataOperations; + this.supportsDots = supportsDots; ENTITY_RESOURCE_TEST_MAP.put(entityTYpe, this); } @@ -735,6 +739,22 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { assertResponse(() -> createEntity(create, ADMIN_AUTH_HEADERS), CONFLICT, ENTITY_ALREADY_EXISTS); } + @Test + void post_entityWithDots_200(TestInfo test) throws HttpResponseException { + if (!supportsDots) { + return; + } + String name = String.format("%s_%s_foo.bar", entityType, test.getDisplayName()); + final K request = createRequest(name, null, null, null); + T entity = createEntity(request, ADMIN_AUTH_HEADERS); + EntityInterface entityInterface = getEntityInterface(entity); + assertEquals(name, entityInterface.getDisplayName()); + String[] split = entityInterface.getFullyQualifiedName().split("\\."); + String actualName = split[split.length - 1]; + assertTrue(actualName.contains("foo_DOT_bar")); + assertTrue(!actualName.contains("foo.bar")); + } + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Common entity tests for PUT operations /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1030,6 +1050,7 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { entity = getEntity(entityInterface.getId(), ADMIN_AUTH_HEADERS); entityInterface = getEntityInterface(entity); + String oldDisplayName = entityInterface.getDisplayName(); // // Add displayName, description, owner, and tags when previously they were null @@ -1043,7 +1064,6 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { // Field changes ChangeDescription change = getChangeDescription(entityInterface.getVersion()); change.getFieldsAdded().add(new FieldChange().withName("description").withNewValue("description")); - change.getFieldsAdded().add(new FieldChange().withName("displayName").withNewValue("displayName")); if (supportsOwner) { entityInterface.setOwner(TEAM_OWNER1); change.getFieldsAdded().add(new FieldChange().withName("owner").withNewValue(TEAM_OWNER1)); @@ -1053,6 +1073,13 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { entityInterface.getTags().add(USER_ADDRESS_TAG_LABEL); change.getFieldsAdded().add(new FieldChange().withName("tags").withNewValue(entityInterface.getTags())); } + if (supportsDots) { + change + .getFieldsUpdated() + .add(new FieldChange().withName("displayName").withOldValue(oldDisplayName).withNewValue("displayName")); + } else { + change.getFieldsAdded().add(new FieldChange().withName("displayName").withNewValue("displayName")); + } entity = patchEntityAndCheck(entity, origJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change); entityInterface = getEntityInterface(entity); @@ -1099,14 +1126,18 @@ public abstract class EntityResourceTest extends CatalogApplicationTest { List removedTags = entityInterface.getTags(); entityInterface.setDescription(null); - entityInterface.setDisplayName(null); + if (!supportsDots) { + entityInterface.setDisplayName(null); + } entityInterface.setOwner(null); entityInterface.setTags(null); // Field changes change = getChangeDescription(entityInterface.getVersion()); change.getFieldsDeleted().add(new FieldChange().withName("description").withOldValue("description1")); - change.getFieldsDeleted().add(new FieldChange().withName("displayName").withOldValue("displayName1")); + if (!supportsDots) { + change.getFieldsDeleted().add(new FieldChange().withName("displayName").withOldValue("displayName1")); + } if (supportsOwner) { change.getFieldsDeleted().add(new FieldChange().withName("owner").withOldValue(USER_OWNER1)); } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/charts/ChartResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/charts/ChartResourceTest.java index 7d3d446645d..7befeee7333 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/charts/ChartResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/charts/ChartResourceTest.java @@ -45,7 +45,7 @@ import org.openmetadata.catalog.util.TestUtils; public class ChartResourceTest extends EntityResourceTest { public ChartResourceTest() { - super(Entity.CHART, Chart.class, ChartList.class, "charts", ChartResource.FIELDS, true, true, true, true); + super(Entity.CHART, Chart.class, ChartList.class, "charts", ChartResource.FIELDS, true, true, true, true, true); } @BeforeAll diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java index 59795b2ed4c..ac6b7129094 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/dashboards/DashboardResourceTest.java @@ -68,6 +68,7 @@ public class DashboardResourceTest extends EntityResourceTest { public TableResourceTest() { - super(Entity.TABLE, Table.class, TableList.class, "tables", TableResource.FIELDS, true, true, true, true); + super(Entity.TABLE, Table.class, TableList.class, "tables", TableResource.FIELDS, true, true, true, true, true); } @BeforeAll @@ -187,6 +189,23 @@ public class TableResourceTest extends EntityResourceTest { assertEquals(expectedFQN, table.getFullyQualifiedName()); } + @Test + void post_tableWithColumnWithDots(TestInfo test) throws IOException { + CreateTable create = createRequest(test); + List columns = new ArrayList<>(); + columns.add(getColumn("col.umn", INT, null)); + TableConstraint constraint = + new TableConstraint().withConstraintType(ConstraintType.UNIQUE).withColumns(List.of(columns.get(0).getName())); + create.setColumns(columns); + create.setTableConstraints(List.of(constraint)); + Table created = createAndCheckEntity(create, ADMIN_AUTH_HEADERS); + Column column = created.getColumns().get(0); + assertTrue(column.getName().equals("col_DOT_umn")); + assertTrue(column.getDisplayName().contains("col.umn")); + assertTrue(column.getFullyQualifiedName().contains("col_DOT_umn")); + assertEquals("col_DOT_umn", created.getTableConstraints().get(0).getColumns().get(0)); + } + public static Column getColumn(String name, ColumnDataType columnDataType, TagLabel tag) { return getColumn(name, columnDataType, null, tag); } @@ -1322,7 +1341,9 @@ public class TableResourceTest extends EntityResourceTest { private static void assertColumn(Column expectedColumn, Column actualColumn) throws HttpResponseException { assertNotNull(actualColumn.getFullyQualifiedName()); - assertEquals(expectedColumn.getName(), actualColumn.getName()); + assertTrue( + expectedColumn.getName().equals(actualColumn.getName()) + || expectedColumn.getName().equals(actualColumn.getDisplayName())); assertEquals(expectedColumn.getDescription(), actualColumn.getDescription()); assertEquals(expectedColumn.getDataType(), actualColumn.getDataType()); assertEquals(expectedColumn.getArrayDataType(), actualColumn.getArrayDataType()); @@ -1460,12 +1481,28 @@ public class TableResourceTest extends EntityResourceTest { assertEquals(createRequest.getTableType(), createdEntity.getTableType()); assertColumns(createRequest.getColumns(), createdEntity.getColumns()); validateDatabase(createRequest.getDatabase(), createdEntity.getDatabase()); - assertEquals(createRequest.getTableConstraints(), createdEntity.getTableConstraints()); + validateTableConstraints(createRequest.getTableConstraints(), createdEntity.getTableConstraints()); TestUtils.validateTags(createRequest.getTags(), createdEntity.getTags()); TestUtils.validateEntityReference(createdEntity.getFollowers()); assertListNotNull(createdEntity.getService(), createdEntity.getServiceType()); } + private void validateTableConstraints(List expected, List actual) { + if (expected == null || actual == null) { + assertEquals(expected, actual); + return; + } + List copy = new ArrayList<>(); + actual.forEach(c -> copy.add(cloneAndUnescape(c))); + assertEquals(expected, copy); + } + + private TableConstraint cloneAndUnescape(TableConstraint constraint) { + return new TableConstraint() + .withConstraintType(constraint.getConstraintType()) + .withColumns(constraint.getColumns().stream().map(e -> e.replace("_DOT_", ".")).collect(Collectors.toList())); + } + @Override public void validateUpdatedEntity(Table updated, CreateTable request, Map authHeaders) throws HttpResponseException { diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java index 7541ac806d0..b2906f6c67a 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/events/WebhookResourceTest.java @@ -64,7 +64,7 @@ public class WebhookResourceTest extends EntityResourceTest expected, List actual) { + if (expected == null || actual == null) { + assertEquals(expected, actual); + return; + } + assertEquals(expected.size(), actual.size()); + int i = 0; + for (Task expectedTask : expected) { + Task actualTask = actual.get(i); + assertTrue( + expectedTask.getName().equals(actualTask.getName()) + || expectedTask.getName().equals(actualTask.getDisplayName())); + i++; + } + } + @Override public void validateUpdatedEntity(Pipeline pipeline, CreatePipeline request, Map authHeaders) throws HttpResponseException { @@ -142,7 +159,7 @@ public class PipelineResourceTest extends EntityResourceTest expectedTasks = (List) expected; List actualTasks = JsonUtils.readObjects(actual.toString(), Task.class); - assertEquals(expectedTasks, actualTasks); + validateTasks(expectedTasks, actualTasks); } else { assertCommonFieldChange(fieldName, expected, actual); } @@ -209,6 +226,17 @@ public class PipelineResourceTest extends EntityResourceTest private static Location location; public PolicyResourceTest() { - super(Entity.POLICY, Policy.class, PolicyList.class, "policies", PolicyResource.FIELDS, false, true, false, false); + super( + Entity.POLICY, + Policy.class, + PolicyList.class, + "policies", + PolicyResource.FIELDS, + false, + true, + false, + false, + false); } @BeforeAll diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/DashboardServiceResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/DashboardServiceResourceTest.java index 14b65d5408f..dc243d923e4 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/DashboardServiceResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/DashboardServiceResourceTest.java @@ -57,6 +57,7 @@ public class DashboardServiceResourceTest extends EntityResourceTest { public RoleResourceTest() { - super(Entity.ROLE, Role.class, RoleList.class, "roles", null, false, false, false, false); + super(Entity.ROLE, Role.class, RoleList.class, "roles", null, false, false, false, false, false); } @Test @@ -141,8 +141,6 @@ public class RoleResourceTest extends EntityResourceTest { public void validateCreatedEntity(Role role, CreateRole createRequest, Map authHeaders) { validateCommonEntityFields( getEntityInterface(role), createRequest.getDescription(), TestUtils.getPrincipal(authHeaders), null); - - assertEquals(createRequest.getDisplayName(), role.getDisplayName()); } @Override diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java index 430c3c968a8..e95df611de0 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java @@ -65,7 +65,7 @@ public class TeamResourceTest extends EntityResourceTest { final Profile PROFILE = new Profile().withImages(new ImageList().withImage(URI.create("http://image.com"))); public TeamResourceTest() { - super(Entity.TEAM, Team.class, TeamList.class, "teams", TeamResource.FIELDS, false, false, false, false); + super(Entity.TEAM, Team.class, TeamList.class, "teams", TeamResource.FIELDS, false, false, false, false, false); } @Test @@ -251,7 +251,6 @@ public class TeamResourceTest extends EntityResourceTest { validateCommonEntityFields( getEntityInterface(team), createRequest.getDescription(), TestUtils.getPrincipal(authHeaders), null); - assertEquals(createRequest.getDisplayName(), team.getDisplayName()); assertEquals(createRequest.getProfile(), team.getProfile()); List expectedUsers = new ArrayList<>(); diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/UserResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/UserResourceTest.java index 67503c1d965..8bab1020ae2 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/UserResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/UserResourceTest.java @@ -83,7 +83,7 @@ public class UserResourceTest extends EntityResourceTest { final Profile PROFILE = new Profile().withImages(new ImageList().withImage(URI.create("http://image.com"))); public UserResourceTest() { - super(Entity.USER, User.class, UserList.class, "users", UserResource.FIELDS, false, false, false, false); + super(Entity.USER, User.class, UserList.class, "users", UserResource.FIELDS, false, false, false, false, false); } @Test diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/topics/TopicResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/topics/TopicResourceTest.java index bc3d0ce7d40..4f7f729aea3 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/topics/TopicResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/topics/TopicResourceTest.java @@ -51,7 +51,7 @@ import org.openmetadata.catalog.util.TestUtils.UpdateType; public class TopicResourceTest extends EntityResourceTest { public TopicResourceTest() { - super(Entity.TOPIC, Topic.class, TopicList.class, "topics", TopicResource.FIELDS, true, true, true, true); + super(Entity.TOPIC, Topic.class, TopicList.class, "topics", TopicResource.FIELDS, true, true, true, true, true); } @Test diff --git a/ingestion-core/src/metadata/_version.py b/ingestion-core/src/metadata/_version.py index 79cb30c0592..c94d8ac4e36 100644 --- a/ingestion-core/src/metadata/_version.py +++ b/ingestion-core/src/metadata/_version.py @@ -7,5 +7,5 @@ Provides metadata version information. from incremental import Version -__version__ = Version("metadata", 0, 9, 0, dev=6) +__version__ = Version("metadata", 0, 9, 0, dev=7) __all__ = ["__version__"]