diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index 96fc7620cfe..76baf60bab5 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -90,6 +90,8 @@ import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response.Status; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.text.RandomStringGenerator; +import org.apache.commons.text.RandomStringGenerator.Builder; import org.apache.http.client.HttpResponseException; import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterAll; @@ -187,14 +189,21 @@ public abstract class EntityResourceTest$\"]"; + "[entityLink must match \"^(?U)<#E::\\w+::[\\w'\\- .&/:+\"\\\\()$#]+>$\"]"; + + // Random unicode string generator to test entity name accepts all the unicode characters + protected static final RandomStringGenerator RANDOM_STRING_GENERATOR = + new Builder().filteredBy(Character::isLetterOrDigit).build(); // Users public static User USER1; @@ -1001,7 +1010,7 @@ public abstract class EntityResourceTest { public TableResourceTest() { super(TABLE, Table.class, TableList.class, "tables", TableResource.FIELDS); - supportedNameCharacters = "_'- .()$"; + supportedNameCharacters = "_'- .()$" + EntityResourceTest.RANDOM_STRING_GENERATOR.generate(1); } public void setupDatabaseSchemas(TestInfo test) throws IOException { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java index 0fe3adbfb62..0750d95d9b2 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/feeds/FeedResourceTest.java @@ -191,7 +191,7 @@ public class FeedResourceTest extends OpenMetadataApplicationTest { // Create thread without addressed to entity in the request CreateThread create = create().withFrom(USER.getName()).withAbout("<>"); // Invalid EntityLink - String failureReason = "[about must match \"^<#E::\\w+::[\\w'\\- .&/:+\"\\\\()$#]+>$\"]"; + String failureReason = "[about must match \"^(?U)<#E::\\w+::[\\w'\\- .&/:+\"\\\\()$#]+>$\"]"; assertResponseContains(() -> createThread(create, AUTH_HEADERS), BAD_REQUEST, failureReason); create.withAbout("<#E::>"); // Invalid EntityLink - missing entityType and entityId diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java index 5b7b0bdde0e..3854a4a5f83 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java @@ -103,7 +103,7 @@ public class GlossaryTermResourceTest extends EntityResourceTest { super(Entity.TYPE, Type.class, TypeList.class, "metadata/types", TypeResource.PROPERTIES); supportsEmptyDescription = false; supportsFieldsQueryParam = false; - supportedNameCharacters = "_"; // No other special characters allowed + supportedNameCharacters = "_" + RANDOM_STRING_GENERATOR.generate(1); // No other special characters allowed } public void setupTypes() throws HttpResponseException { @@ -72,9 +72,9 @@ public class TypeResourceTest extends EntityResourceTest { @Test public void post_entityCreateWithInvalidName_400() { // Names can't start with capital letter, can't have space, hyphen, apostrophe - String[] tests = {"Abcd", "a bc", "a-bc", "a'b"}; + String[] tests = {"a bc", "a-bc", "a'b"}; - String error = "[name must match \"^[a-z][\\w]+$\"]"; + String error = "[name must match \"^(?U)[\\w]+$\"]"; CreateType create = createRequest("placeHolder", "", "", null); for (String test : tests) { LOG.info("Testing with the name {}", test); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java index 8d8cce2d9e2..702a43d32de 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java @@ -483,12 +483,13 @@ public final class TestUtils { if (expected == null && actual == null) { return; } - if (listOrEmpty(expected).isEmpty()) { + expected = listOrEmpty(expected); + actual = listOrEmpty(actual); + if (expected.isEmpty()) { return; } - for (UUID id : listOrEmpty(expected)) { - actual = listOrEmpty(actual); - assertEquals(expected.size(), actual.size()); + assertEquals(expected.size(), actual.size()); + for (UUID id : expected) { assertNotNull(actual.stream().filter(entity -> entity.getId().equals(id)).findAny().orElse(null)); } validateEntityReferences(actual); diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/location.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/location.json index df21fb6c6ec..eba0e86322a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/location.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/location.json @@ -12,7 +12,7 @@ "type": "string", "minLength": 1, "maxLength": 128, - "pattern": "^[\\w'\\-./]+$" + "pattern": "^(?U)[\\w'\\-./]+$" }, "locationType": { "javaType": "org.openmetadata.schema.type.LocationType", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json index 23bbd240c3c..e46c80d3c1c 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json @@ -15,7 +15,7 @@ "type": "string", "minLength": 1, "maxLength": 128, - "pattern": "^[\\w'\\- ./()$]+$" + "pattern": "^(?U)[\\w'\\- ./()$]+$" }, "profileSampleType": { "description": "Type of Profile Sample (percentage or rows)", @@ -162,7 +162,7 @@ "columnName": { "description": "Local name (not fully qualified name) of the column. ColumnName is `-` when the column is not named in struct dataType. For example, BigQuery supports struct with unnamed fields.", "type": "string", - "pattern": "^[\\w'\\-\" .+#]+$", + "pattern": "^(?U)[\\w'\\-\" .+#]+$", "minLength": 1, "maxLength": 128 }, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/events/webhook.json b/openmetadata-spec/src/main/resources/json/schema/entity/events/webhook.json index 30543eca73a..1d3bc418814 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/events/webhook.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/events/webhook.json @@ -11,7 +11,7 @@ "type": "string", "minLength": 1, "maxLength": 128, - "pattern": "^[\\w'\\-.]+$" + "pattern": "^(?U)[\\w'\\-.]+$" } }, "properties": { diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/teams/user.json b/openmetadata-spec/src/main/resources/json/schema/entity/teams/user.json index 857325fb252..32d456f7904 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/teams/user.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/teams/user.json @@ -12,7 +12,7 @@ "type": "string", "minLength": 1, "maxLength": 64, - "pattern": "([\\w\\-.]|[^@])+$" + "pattern": "(?U)([\\w\\-.]|[^@])+$" }, "authenticationMechanism": { "type": "object", diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/type.json b/openmetadata-spec/src/main/resources/json/schema/entity/type.json index 1ba61128efc..b47245dfe86 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/type.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/type.json @@ -10,7 +10,7 @@ "entityName": { "description": "Name of the property or entity types. Note a property name must be unique for an entity. Property name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.", "type": "string", - "pattern": "^[a-z][\\w]+$" + "pattern": "^(?U)[\\w]+$" }, "category": { "description": "Metadata category to which a type belongs to.", @@ -26,11 +26,6 @@ } ] }, - "propertyName": { - "description": "Name of the entity property. Note a property name must be unique for an entity. Property name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.", - "type": "string", - "pattern": "^[a-z][\\w]+$" - }, "customProperty": { "description": "Type used for adding custom property to an entity to extend it.", "type": "object", @@ -38,7 +33,7 @@ "properties": { "name": { "description": "Name of the entity property. Note a property name must be unique for an entity. Property name must follow camelCase naming adopted by openMetadata - must start with lower case with no space, underscore, or dots.", - "$ref": "#/definitions/propertyName" + "$ref": "#/definitions/entityName" }, "description": { "$ref": "../type/basic.json#/definitions/markdown" diff --git a/openmetadata-spec/src/main/resources/json/schema/type/basic.json b/openmetadata-spec/src/main/resources/json/schema/type/basic.json index 410b7255bbb..0434177901e 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/basic.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/basic.json @@ -93,14 +93,14 @@ "entityLink": { "description": "Link to an entity or field within an entity using this format `<#E::{entities}::{entityType}::{field}::{arrayFieldName}::{arrayFieldValue}`.", "type": "string", - "pattern": "^<#E::\\w+::[\\w'\\- .&/:+\"\\\\()$#]+>$" + "pattern": "^(?U)<#E::\\w+::[\\w'\\- .&/:+\"\\\\()$#]+>$" }, "entityName": { "description": "Name that identifies a entity.", "type": "string", "minLength": 1, "maxLength": 128, - "pattern": "^[\\w'\\- .&]+$" + "pattern": "^(?U)[\\w'\\- .&]+$" }, "fullyQualifiedEntityName": { "description": "A unique name that identifies an entity. Example for table 'DatabaseService:Database:Table'.", diff --git a/scripts/datamodel_generation.py b/scripts/datamodel_generation.py index 41c29652da5..bbacb86ec03 100644 --- a/scripts/datamodel_generation.py +++ b/scripts/datamodel_generation.py @@ -18,6 +18,15 @@ from a configured secrets' manager. import datamodel_code_generator.model.pydantic from datamodel_code_generator.imports import Import +UNICODE_REGEX_REPLACEMENT_FILE_PATHS = [ + "ingestion/src/metadata/generated/schema/entity/data/location.py", + "ingestion/src/metadata/generated/schema/entity/data/table.py", + "ingestion/src/metadata/generated/schema/entity/events/webhook.py", + "ingestion/src/metadata/generated/schema/entity/teams/user.py", + "ingestion/src/metadata/generated/schema/entity/type.py", + "ingestion/src/metadata/generated/schema/type/basic.py", +] + datamodel_code_generator.model.pydantic.types.IMPORT_SECRET_STR = Import.from_full_path( "metadata.ingestion.models.custom_pydantic.CustomSecretStr" ) @@ -28,3 +37,10 @@ args = "--input openmetadata-spec/src/main/resources/json/schema --input-file-ty " " ) main(args) + +for file_path in UNICODE_REGEX_REPLACEMENT_FILE_PATHS: + with open(file_path, "r", encoding="UTF-8") as file_: + content = file_.read() + content = content.replace("(?U)", "(?u)") + with open(file_path, "w", encoding="UTF-8") as file_: + file_.write(content)