From abcdc4e3d6be1b0d25f71963e37963b0a1a7424e Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Fri, 29 Aug 2025 17:28:29 +0200 Subject: [PATCH] MINOR - Domain Independent DP Rule (#23067) * MINOR - Domain Independent DP Rule * handle DP * Handle DP * add migration * improve rule mgmt * improve rule mgmt * add test for bulk op * fix test * handle in bulk --------- Co-authored-by: sonika-shah <58761340+sonika-shah@users.noreply.github.com> --- .../native/1.10.0/mysql/schemaChanges.sql | 19 + .../native/1.10.0/postgres/schemaChanges.sql | 21 + .../ometa/test_ometa_domains_api.py | 9 +- .../service/jdbi3/DataProductRepository.java | 149 +++-- .../service/jdbi3/EntityRepository.java | 37 -- .../openmetadata/service/rules/LogicOps.java | 61 +++ .../rules/RuleValidationException.java | 4 + .../data/settings/entityRulesSettings.json | 7 + .../service/resources/EntityResourceTest.java | 19 +- .../domains/DataProductResourceTest.java | 169 ++++++ .../service/rules/RuleEngineTests.java | 518 +++++++++++++++++- 11 files changed, 907 insertions(+), 106 deletions(-) create mode 100644 bootstrap/sql/migrations/native/1.10.0/mysql/schemaChanges.sql create mode 100644 bootstrap/sql/migrations/native/1.10.0/postgres/schemaChanges.sql diff --git a/bootstrap/sql/migrations/native/1.10.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.10.0/mysql/schemaChanges.sql new file mode 100644 index 00000000000..3b34c506421 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.10.0/mysql/schemaChanges.sql @@ -0,0 +1,19 @@ +-- Add "Data Product Domain Validation" rule to existing entityRulesSettings configuration +UPDATE openmetadata_settings +SET json = JSON_ARRAY_APPEND( + json, + '$.entitySemantics', + JSON_OBJECT( + 'name', 'Data Product Domain Validation', + 'description', 'Validates that Data Products assigned to an entity match the entity''s domains.', + 'rule', '{"validateDataProductDomainMatch":[{"var":"dataProducts"},{"var":"domains"}]}', + 'enabled', true, + 'provider', 'system' + ) +) +WHERE configType = 'entityRulesSettings' + AND JSON_EXTRACT(json, '$.entitySemantics') IS NOT NULL + AND NOT JSON_CONTAINS( + JSON_EXTRACT(json, '$.entitySemantics[*].name'), + JSON_QUOTE('Data Product Domain Validation') + ); \ No newline at end of file diff --git a/bootstrap/sql/migrations/native/1.10.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.10.0/postgres/schemaChanges.sql new file mode 100644 index 00000000000..6911da395b6 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.10.0/postgres/schemaChanges.sql @@ -0,0 +1,21 @@ +-- Add "Data Product Domain Validation" rule to existing entityRulesSettings configuration +UPDATE openmetadata_settings +SET json = jsonb_set( + json, + '{entitySemantics}', + (json->'entitySemantics') || jsonb_build_object( + 'name', 'Data Product Domain Validation', + 'description', 'Validates that Data Products assigned to an entity match the entity''s domains.', + 'rule', '{"validateDataProductDomainMatch":[{"var":"dataProducts"},{"var":"domains"}]}', + 'enabled', true, + 'provider', 'system' + )::jsonb, + true +) +WHERE configtype = 'entityRulesSettings' + AND json->'entitySemantics' IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM jsonb_array_elements(json->'entitySemantics') AS rule + WHERE rule->>'name' = 'Data Product Domain Validation' + ); \ No newline at end of file diff --git a/ingestion/tests/integration/ometa/test_ometa_domains_api.py b/ingestion/tests/integration/ometa/test_ometa_domains_api.py index a773c00fb81..374cc5b53b7 100644 --- a/ingestion/tests/integration/ometa/test_ometa_domains_api.py +++ b/ingestion/tests/integration/ometa/test_ometa_domains_api.py @@ -201,7 +201,14 @@ class OMetaDomainTest(TestCase): def test_add_remove_assets_to_data_product(self): """We can add assets to a data product""" - self.metadata.create_or_update(data=self.create_domain) + domain: Domain = self.metadata.create_or_update(data=self.create_domain) + domains_ref = EntityReferenceList( + root=[EntityReference(id=domain.id, type="domain")] + ) + # Make sure the dashboard belongs to the data product domain! + self.metadata.patch_domain( + entity=Dashboard, source=self.dashboard, domains=domains_ref + ) data_product: DataProduct = self.metadata.create_or_update( data=self.create_data_product ) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java index 6b194c3cf9e..81c14ccc1e2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java @@ -31,7 +31,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.schema.EntityInterface; @@ -50,6 +49,8 @@ import org.openmetadata.schema.type.change.ChangeSource; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.domains.DataProductResource; +import org.openmetadata.service.rules.RuleEngine; +import org.openmetadata.service.rules.RuleValidationException; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.LineageUtil; @@ -214,34 +215,73 @@ public class DataProductRepository extends EntityRepository { BulkOperationResult result = new BulkOperationResult().withStatus(ApiStatus.SUCCESS).withDryRun(false); List success = new ArrayList<>(); + List failed = new ArrayList<>(); EntityUtil.populateEntityReferences(request.getAssets()); - for (EntityReference ref : request.getAssets()) { - result.setNumberOfRowsProcessed(result.getNumberOfRowsProcessed() + 1); + // Get the data product reference for validation + DataProduct dataProduct = find(entityId, ALL); + EntityReference dataProductRef = dataProduct.getEntityReference(); - removeCrossDomainDataProducts(ref, relationship); - - if (isAdd) { - addRelationship(entityId, ref.getId(), fromEntity, ref.getType(), relationship); - } else { - deleteRelationship(entityId, fromEntity, ref.getId(), ref.getType(), relationship); - } - - success.add(new BulkResponse().withRequest(ref)); - result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); - - searchRepository.updateEntity(ref); + // Fetch all asset entities in bulk for validation + List assetEntities = new ArrayList<>(); + if (isAdd && !request.getAssets().isEmpty()) { + assetEntities = Entity.getEntities(request.getAssets(), "domains,dataProducts", ALL); } - result.withSuccessRequest(success); + for (int i = 0; i < request.getAssets().size(); i++) { + EntityReference ref = request.getAssets().get(i); + result.setNumberOfRowsProcessed(result.getNumberOfRowsProcessed() + 1); - // Create a Change Event on successful addition/removal of assets - if (result.getStatus().equals(ApiStatus.SUCCESS)) { + try { + if (isAdd) { + EntityInterface assetEntity = assetEntities.get(i); + validateAssetDataProductAssignment(assetEntity, dataProductRef); + addRelationship(entityId, ref.getId(), fromEntity, ref.getType(), relationship); + } else { + deleteRelationship(entityId, fromEntity, ref.getId(), ref.getType(), relationship); + } + + success.add(new BulkResponse().withRequest(ref)); + result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); + + searchRepository.updateEntity(ref); + } catch (RuleValidationException e) { + LOG.warn( + "Validation failed for asset {} in bulk operation: {}", ref.getId(), e.getMessage()); + failed.add(new BulkResponse().withRequest(ref).withMessage(e.getMessage())); + result.setNumberOfRowsFailed(result.getNumberOfRowsFailed() + 1); + result.setStatus(ApiStatus.PARTIAL_SUCCESS); + } catch (Exception e) { + LOG.error( + "Unexpected error during bulk operation for asset {}: {}", + ref.getId(), + e.getMessage(), + e); + failed.add( + new BulkResponse().withRequest(ref).withMessage("Internal error: " + e.getMessage())); + result.setNumberOfRowsFailed(result.getNumberOfRowsFailed() + 1); + result.setStatus(ApiStatus.PARTIAL_SUCCESS); + } + } + + result.withSuccessRequest(success).withFailedRequest(failed); + + // If all operations failed, mark as failure + if (success.isEmpty() && !failed.isEmpty()) { + result.setStatus(ApiStatus.FAILURE); + } + + // Create a Change Event on successful operations + if (!success.isEmpty()) { EntityInterface entityInterface = Entity.getEntity(fromEntity, entityId, "id", ALL); + List successfulAssets = new ArrayList<>(); + for (BulkResponse response : success) { + successfulAssets.add((EntityReference) response.getRequest()); + } ChangeDescription change = addBulkAddRemoveChangeDescription( - entityInterface.getVersion(), isAdd, request.getAssets(), null); + entityInterface.getVersion(), isAdd, successfulAssets, null); ChangeEvent changeEvent = getChangeEvent(entityInterface, change, fromEntity, entityInterface.getVersion()); Entity.getCollectionDAO().changeEventDAO().insert(JsonUtils.pojoToJson(changeEvent)); @@ -250,50 +290,39 @@ public class DataProductRepository extends EntityRepository { return result; } - private void removeCrossDomainDataProducts(EntityReference ref, Relationship relationship) { - EntityReference domain = - getFromEntityRef(ref.getId(), ref.getType(), relationship, DOMAIN, false); - List dataProducts = getDataProducts(ref.getId(), ref.getType()); + /** + * Validates that an asset can be assigned to a data product according to configured rules. + * This method leverages the RuleEngine to validate domain matching rules that are enabled. + * + * @param assetEntity The asset entity interface (pre-fetched with domains,dataProducts) + * @param dataProductRef The data product entity reference + * @throws RuleValidationException if validation fails + */ + private void validateAssetDataProductAssignment( + EntityInterface assetEntity, EntityReference dataProductRef) { + try { - if (!dataProducts.isEmpty() && domain != null) { - // Map dataProduct -> domain - Map associatedDomains = - daoCollection - .relationshipDAO() - .findFromBatch( - dataProducts.stream() - .map(dp -> dp.getId().toString()) - .collect(Collectors.toList()), - relationship.ordinal(), - DOMAIN) - .stream() - .collect( - Collectors.toMap( - rec -> UUID.fromString(rec.getToId()), - rec -> UUID.fromString(rec.getFromId()))); + List currentDataProducts = listOrEmpty(assetEntity.getDataProducts()); + List updatedDataProducts = new ArrayList<>(currentDataProducts); + updatedDataProducts.add(dataProductRef); - List dataProductsToDelete = - dataProducts.stream() - .filter( - dataProduct -> { - UUID associatedDomainId = associatedDomains.get(dataProduct.getId()); - return associatedDomainId != null && !associatedDomainId.equals(domain.getId()); - }) - .collect(Collectors.toList()); + assetEntity.setDataProducts(updatedDataProducts); + RuleEngine.getInstance().evaluate(assetEntity, true, false); - if (!dataProductsToDelete.isEmpty()) { - daoCollection - .relationshipDAO() - .bulkRemoveFromRelationship( - dataProductsToDelete.stream() - .map(EntityReference::getId) - .collect(Collectors.toList()), - ref.getId(), - DATA_PRODUCT, - ref.getType(), - relationship.ordinal()); - LineageUtil.removeDataProductsLineage(ref.getId(), ref.getType(), dataProductsToDelete); - } + } catch (RuleValidationException e) { + // Re-throw validation exceptions with context about the bulk operation + throw new RuleValidationException( + String.format( + "Cannot assign asset '%s' (type: %s) to data product '%s': %s", + assetEntity.getName(), + assetEntity.getEntityReference().getType(), + dataProductRef.getName(), + e.getMessage())); + } catch (Exception e) { + LOG.warn( + "Error during asset data product validation for asset {}: {}", + assetEntity.getId(), + e.getMessage()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index df7e0e1c3a5..affd8e2a805 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -2567,38 +2567,6 @@ public abstract class EntityRepository { return RestUtil.getHref(uriInfo, collectionPath, id); } - private void removeCrossDomainDataProducts(List removedDomains, T entity) { - if (!supportsDataProducts) { - return; - } - - List entityDataProducts = entity.getDataProducts(); - if (entityDataProducts == null || nullOrEmpty(removedDomains)) { - // If no domains are being removed, nothing to do - return; - } - - for (EntityReference domain : removedDomains) { - // Fetch domain data products - List domainDataProductIds = - daoCollection - .relationshipDAO() - .findToIds(domain.getId(), DOMAIN, Relationship.HAS.ordinal(), Entity.DATA_PRODUCT); - - entityDataProducts.removeIf( - dataProduct -> { - boolean isNotDomainDataProduct = !domainDataProductIds.contains(dataProduct.getId()); - if (isNotDomainDataProduct) { - LOG.info( - "Removing data product {} from entity {}", - dataProduct.getFullyQualifiedName(), - entity.getEntityReference().getType()); - } - return isNotDomainDataProduct; - }); - } - } - @Transaction public final PutResponse restoreEntity(String updatedBy, UUID id) { // If an entity being restored contains other **deleted** children entities, restore them @@ -4103,8 +4071,6 @@ public abstract class EntityRepository { entityReferenceMatch)) { updateDomains(original, origDomains, updatedDomains); updated.setDomains(updatedDomains); - // Clean up data products associated with the domains we are removing - removeCrossDomainDataProducts(removedDomains, updated); } else { updated.setDomains(original.getDomains()); } @@ -4169,8 +4135,6 @@ public abstract class EntityRepository { entityReferenceMatch)) { updateDomains(original, origDomains, updatedDomains); updated.setDomains(updatedDomains); - // Clean up data products associated with the domains we are removing - removeCrossDomainDataProducts(removedDomains, updated); } else { updated.setDomains(original.getDomains()); } @@ -4207,7 +4171,6 @@ public abstract class EntityRepository { updatedDataProducts, true, entityReferenceListMatch)) { - removeCrossDomainDataProducts(removedDomains, updated); updatedDataProducts = listOrEmpty(updated.getDataProducts()); } updateFromRelationships( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/rules/LogicOps.java b/openmetadata-service/src/main/java/org/openmetadata/service/rules/LogicOps.java index 97423d6c8ed..fb65def68fd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/rules/LogicOps.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/rules/LogicOps.java @@ -11,11 +11,18 @@ import io.github.jamsesso.jsonlogic.evaluator.JsonLogicEvaluationException; import io.github.jamsesso.jsonlogic.evaluator.JsonLogicEvaluator; import io.github.jamsesso.jsonlogic.evaluator.JsonLogicExpression; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.schema.entity.domains.DataProduct; import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; public class LogicOps { @@ -107,6 +114,60 @@ public class LogicOps { return true; }); + // Domain validation for Data Product assignment + // This validates that entities being assigned to Data Products have matching domains + jsonLogic.addOperation( + "validateDataProductDomainMatch", + (args) -> { + if (nullOrEmpty(args) || args.length < 2) { + return true; // If no data products or domains, validation passes + } + + List dataProducts = + JsonUtils.convertValue(args[0], new TypeReference>() {}); + List domains = + JsonUtils.convertValue(args[1], new TypeReference>() {}); + + if (nullOrEmpty(dataProducts)) { + return true; // If no data products, validation passes + } else if (nullOrEmpty(domains)) { + return false; // If data products but no domains, validation fails + } + + // Convert entity domains to a Set of UUIDs for efficient lookup + Set entityDomainIds = + domains.stream().map(EntityReference::getId).collect(Collectors.toSet()); + + // Get all data product entities in bulk instead of using a loop + try { + List dpEntities = + Entity.getEntities(dataProducts, "domains", Include.NON_DELETED); + + for (DataProduct dpEntity : dpEntities) { + List dpDomains = dpEntity.getDomains(); + + if (nullOrEmpty(dpDomains)) { + continue; // If data product has no domains, skip validation + } + + // Convert data product domains to a Set of UUIDs for efficient comparison + Set dpDomainIds = + dpDomains.stream().map(EntityReference::getId).collect(Collectors.toSet()); + + // Use Collections.disjoint for O(n+m) performance instead of O(n*m) + boolean hasMatchingDomain = !Collections.disjoint(entityDomainIds, dpDomainIds); + + if (!hasMatchingDomain) { + return false; // Domain mismatch found + } + } + } catch (Exception e) { + // If we can't fetch the data products, fail validation for safety + return false; + } + return true; // All data products have matching domains + }); + // Example: {"filterReferenceByType":[{"var":"owner"},"team"]} jsonLogic.addOperation( "filterReferenceByType", diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/rules/RuleValidationException.java b/openmetadata-service/src/main/java/org/openmetadata/service/rules/RuleValidationException.java index 7101de83628..6d913459bc0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/rules/RuleValidationException.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/rules/RuleValidationException.java @@ -12,6 +12,10 @@ public class RuleValidationException extends IllegalArgumentException { super(formatMessage(rule, message)); } + public RuleValidationException(String message) { + super(message); + } + public RuleValidationException(List rules, String message) { super(formatMessage(rules, message)); } diff --git a/openmetadata-service/src/main/resources/json/data/settings/entityRulesSettings.json b/openmetadata-service/src/main/resources/json/data/settings/entityRulesSettings.json index 35b3063255f..36659ecc878 100644 --- a/openmetadata-service/src/main/resources/json/data/settings/entityRulesSettings.json +++ b/openmetadata-service/src/main/resources/json/data/settings/entityRulesSettings.json @@ -29,6 +29,13 @@ "enabled": false, "entityType": "table", "provider": "system" + }, + { + "name": "Data Product Domain Validation", + "description": "Validates that Data Products assigned to an entity match the entity's domains.", + "rule": "{\"validateDataProductDomainMatch\":[{\"var\":\"dataProducts\"},{\"var\":\"domains\"}]}", + "enabled": true, + "provider": "system" } ] } \ No newline at end of file diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index 61c3cfc8871..ecbaa34d125 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -292,9 +292,6 @@ public abstract class EntityResourceTest|]|:[^:<>|])+(?:::(?:[^:<>|]|:[^:<>|])+)*>$\"]"; - public static final String MULTIDOMAIN_RULE_ERROR = - "Rule [Multiple Domains are not allowed] validation failed: Entity does not satisfy the rule. Rule context: " - + "By default, we only allow entities to be assigned to a single domain, except for Users and Teams."; // Random unicode string generator to test entity name accepts all the unicode characters protected static final RandomStringGenerator RANDOM_STRING_GENERATOR = @@ -681,6 +678,16 @@ public abstract class EntityResourceTest { - if (MULTI_DOMAIN_RULE.equals(rule.getName())) { + if (ruleName.equals(rule.getName())) { rule.setEnabled(enable); } }); @@ -1203,8 +1210,8 @@ public abstract class EntityResourceTest patchEntityAndCheck(entity, originalJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change), - NOT_FOUND, - String.format("dataProduct instance for %s not found", dataProductReference.getId())); + BAD_REQUEST, + "Rule [Data Product Domain Validation] validation failed: Entity does not satisfy the rule. Rule context: Validates that Data Products assigned to an entity match the entity's domains."); } @Test diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java index 229918db14a..24f0b93e0d8 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/domains/DataProductResourceTest.java @@ -27,8 +27,11 @@ import org.openmetadata.schema.entity.data.Topic; import org.openmetadata.schema.entity.domains.DataProduct; import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.entity.type.Style; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.api.BulkAssets; +import org.openmetadata.schema.type.api.BulkOperationResult; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; @@ -340,4 +343,170 @@ public class DataProductResourceTest extends EntityResourceTest new IllegalStateException( "No glossary validation rule found for tables. Review the entityRulesSettings.json file.")); + // Enable it for the test + glossaryRule.setEnabled(true); // No glossary terms, should pass RuleEngine.getInstance().evaluate(table, List.of(glossaryRule), false, false); @@ -314,7 +326,7 @@ public class RuleEngineTests extends OpenMetadataApplicationTest { CreateDomain createDomain = domainResourceTest - .createRequest(test) + .createRequest(test.getDisplayName() + "_Data") .withName("Data") .withDescription("Data domain") .withOwners(List.of(USER1_REF)); @@ -352,7 +364,7 @@ public class RuleEngineTests extends OpenMetadataApplicationTest { dataContractResourceTest.createDataContract(createContractForTable); - // Table does indeed blow up + // Table does indeed blow up when evaluated against data contract rules assertThrows( RuleValidationException.class, () -> RuleEngine.getInstance().evaluate(tableWithContract, false, true)); @@ -463,4 +475,506 @@ public class RuleEngineTests extends OpenMetadataApplicationTest { table.withTags(List.of(TIER1_TAG_LABEL)); RuleEngine.getInstance().evaluate(table, List.of(piiRule), false, false); } + + /** + * Helper method to create a real Domain entity for testing + */ + Domain createTestDomain(String name, TestInfo test) throws IOException { + CreateDomain createDomain = + domainResourceTest + .createRequest(test.getDisplayName() + "_" + name) + .withDescription("Test domain: " + name); + return domainResourceTest.createEntity(createDomain, ADMIN_AUTH_HEADERS); + } + + /** + * Helper method to create a real DataProduct entity for testing + */ + DataProduct createTestDataProduct(String name, List domains, TestInfo test) + throws IOException { + String entityName = test.getDisplayName() + "_" + name; + + CreateDataProduct createDataProduct = + dataProductResourceTest + .createRequest(entityName) + .withName(entityName) + .withDescription("Test data product: " + name); + + if (domains != null && !domains.isEmpty()) { + createDataProduct.withDomains( + domains.stream().map(domain -> domain.getFullyQualifiedName()).toList()); + } + + return dataProductResourceTest.createEntity(createDataProduct, ADMIN_AUTH_HEADERS); + } + + /** + * Helper method to get the Data Product Domain Validation rule + */ + SemanticsRule getDataProductDomainRule() { + List rules = + SettingsCache.getSetting(SettingsType.ENTITY_RULES_SETTINGS, EntityRulesSettings.class) + .getEntitySemantics(); + + return rules.stream() + .filter(rule -> DATA_PRODUCT_DOMAIN_RULE.equals(rule.getName())) + .findFirst() + .orElseThrow( + () -> + new IllegalStateException( + "Data Product Domain Validation rule not found. Review the entityRulesSettings.json file.")); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataProductDomainValidation_RuleEnabled_MatchingDomains_ShouldPass(TestInfo test) + throws IOException { + // Create domains + Domain dataDomain = createTestDomain("Data", test); + Domain engineeringDomain = createTestDomain("Engineering", test); + + // Create data products with matching domains + DataProduct dataProduct1 = createTestDataProduct("Product1", List.of(dataDomain), test); + DataProduct dataProduct2 = createTestDataProduct("Product2", List.of(engineeringDomain), test); + + EntityResourceTest.toggleMultiDomainSupport(false); + // Create table with domains that match the data products + CreateTable createTable = + tableResourceTest + .createRequest(test) + .withDomains( + List.of( + dataDomain.getFullyQualifiedName(), engineeringDomain.getFullyQualifiedName())) + .withDataProducts( + List.of( + dataProduct1.getFullyQualifiedName(), dataProduct2.getFullyQualifiedName())); + + Table table = tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS); + + SemanticsRule rule = getDataProductDomainRule(); + + // Should pass validation as entity domains match data product domains + RuleEngine.getInstance().evaluate(table, List.of(rule), false, false); + EntityResourceTest.toggleMultiDomainSupport(true); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataProductDomainValidation_RuleEnabled_NoMatchingDomains_ShouldFail(TestInfo test) + throws IOException { + // Create domains + Domain dataDomain = createTestDomain("Data", test); + Domain engineeringDomain = createTestDomain("Engineering", test); + Domain marketingDomain = createTestDomain("Marketing", test); + + // Create data products with different domains than entity + DataProduct dataProduct1 = createTestDataProduct("Product1", List.of(dataDomain), test); + DataProduct dataProduct2 = createTestDataProduct("Product2", List.of(engineeringDomain), test); + + // Create table with domains that don't match the data products + CreateTable createTable = + tableResourceTest + .createRequest(test) + .withDomains(List.of(marketingDomain.getFullyQualifiedName())) + .withDataProducts( + List.of( + dataProduct1.getFullyQualifiedName(), dataProduct2.getFullyQualifiedName())); + + // Should fail validation as entity domains don't match data product domains + assertResponse( + () -> tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS), + Response.Status.BAD_REQUEST, + DATA_PRODUCT_DOMAIN_RULE_EXC); + + // But I can disable the rule and create it + EntityResourceTest.toggleRule(DATA_PRODUCT_DOMAIN_RULE, false); + tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS); + EntityResourceTest.toggleRule(DATA_PRODUCT_DOMAIN_RULE, true); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataProductDomainValidation_RuleEnabled_MixedMatching_ShouldFail(TestInfo test) + throws IOException { + // Create domains + Domain dataDomain = createTestDomain("Data", test); + Domain engineeringDomain = createTestDomain("Engineering", test); + Domain marketingDomain = createTestDomain("Marketing", test); + + // Create data products - one matches, one doesn't + DataProduct matchingDataProduct = + createTestDataProduct("MatchingProduct", List.of(dataDomain), test); + DataProduct nonMatchingDataProduct = + createTestDataProduct("NonMatchingProduct", List.of(marketingDomain), test); + + // Create table with some matching and some non-matching domains + CreateTable createTable = + tableResourceTest + .createRequest(test) + .withDomains( + List.of( + dataDomain.getFullyQualifiedName(), engineeringDomain.getFullyQualifiedName())) + .withDataProducts( + List.of( + matchingDataProduct.getFullyQualifiedName(), + nonMatchingDataProduct.getFullyQualifiedName())); + + // Should fail during entity creation due to rule validation + assertResponse( + () -> tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS), + Response.Status.BAD_REQUEST, + "Entity does not satisfy multiple rules\n" + + "Rule [Multiple Domains are not allowed] validation failed: Rule context: By default, we only allow entities to be assigned to a single domain, except for Users and Teams.\n" + + "Rule [Data Product Domain Validation] validation failed: Rule context: Validates that Data Products assigned to an entity match the entity's domains."); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataProductDomainValidation_RuleDisabled_DifferentDomains_ShouldPass(TestInfo test) + throws IOException { + // Store original rule state + SemanticsRule originalRule = getDataProductDomainRule(); + Boolean originalEnabled = originalRule.getEnabled(); + + try { + // Disable the rule using the system settings + EntityResourceTest.toggleRule(DATA_PRODUCT_DOMAIN_RULE, false); + + // Create domains + Domain dataDomain = createTestDomain("Data", test); + Domain engineeringDomain = createTestDomain("Engineering", test); + Domain marketingDomain = createTestDomain("Marketing", test); + + // Create data products with completely different domains than entity + DataProduct dataProduct1 = createTestDataProduct("Product1", List.of(dataDomain), test); + DataProduct dataProduct2 = + createTestDataProduct("Product2", List.of(engineeringDomain), test); + + // Create table with domains that don't match the data products + CreateTable createTable = + tableResourceTest + .createRequest(test) + .withDomains(List.of(marketingDomain.getFullyQualifiedName())) + .withDataProducts( + List.of( + dataProduct1.getFullyQualifiedName(), dataProduct2.getFullyQualifiedName())); + + Table table = tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS); + + SemanticsRule rule = getDataProductDomainRule(); + + // Should pass validation when rule is disabled, even with non-matching domains + RuleEngine.getInstance().evaluate(table, List.of(rule), false, false); + } finally { + // Restore original rule state + EntityResourceTest.toggleRule(DATA_PRODUCT_DOMAIN_RULE, originalEnabled); + } + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataProductDomainValidation_EdgeCase_EntityWithNullDomains_ShouldHandleGracefully( + TestInfo test) throws IOException { + // Create domain and data product + Domain dataDomain = createTestDomain("Data", test); + DataProduct dataProduct = createTestDataProduct("Product1", List.of(dataDomain), test); + + // Create table with no domains but assigned to data products with domains + CreateTable createTable = + tableResourceTest + .createRequest(test) + .withDomains(null) // No domains + .withDataProducts(List.of(dataProduct.getFullyQualifiedName())); + + // Should fail during entity creation due to rule validation + assertResponse( + () -> tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS), + Response.Status.BAD_REQUEST, + DATA_PRODUCT_DOMAIN_RULE_EXC); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataProductDomainValidation_EdgeCase_DataProductWithNullDomains_ShouldHandleGracefully( + TestInfo test) throws IOException { + // Create domain for entity and data product with no domains + Domain dataDomain = createTestDomain("Data", test); + DataProduct dataProduct = createTestDataProduct("Product1", null, test); + + // Create table with domains but assigned to data products with no domains + CreateTable createTable = + tableResourceTest + .createRequest(test) + .withDomains(List.of(dataDomain.getFullyQualifiedName())) + .withDataProducts(List.of(dataProduct.getFullyQualifiedName())); + + // Should fail during entity creation due to rule validation + assertResponse( + () -> tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS), + Response.Status.BAD_REQUEST, + DATA_PRODUCT_DOMAIN_RULE_EXC); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataProductDomainValidation_EdgeCase_NoDataProducts_ShouldAlwaysPass(TestInfo test) + throws IOException { + // Create domain for entity + Domain dataDomain = createTestDomain("Data", test); + + // Create table with domains but no data products assigned + CreateTable createTable = + tableResourceTest + .createRequest(test) + .withDomains(List.of(dataDomain.getFullyQualifiedName())) + .withDataProducts(null); // No data products + + Table table = tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS); + + SemanticsRule rule = getDataProductDomainRule(); + + // Should always pass validation when entity has no data products + RuleEngine.getInstance().evaluate(table, List.of(rule), false, false); + + // Also test with empty list + CreateTable createTableEmpty = + tableResourceTest + .createRequest(test.getDisplayName() + "_empty") + .withDomains(List.of(dataDomain.getFullyQualifiedName())) + .withDataProducts(List.of()); // Empty data products + + Table tableEmpty = tableResourceTest.createEntity(createTableEmpty, ADMIN_AUTH_HEADERS); + RuleEngine.getInstance().evaluate(tableEmpty, List.of(rule), false, false); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataProductDomainValidation_EdgeCase_NoDomains_ShouldHandleGracefully(TestInfo test) + throws IOException { + // Create data product with no domains + DataProduct dataProduct = createTestDataProduct("Product1", null, test); + + // Create table with no domains assigned to data products with no domains + CreateTable createTable = + tableResourceTest + .createRequest(test) + .withDomains(null) // No domains + .withDataProducts(List.of(dataProduct.getFullyQualifiedName())); + + // Should fail during entity creation due to rule validation + assertResponse( + () -> tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS), + Response.Status.BAD_REQUEST, + DATA_PRODUCT_DOMAIN_RULE_EXC); + + // Also test with empty domain lists + CreateTable createTableEmpty = + tableResourceTest + .createRequest(test.getDisplayName() + "_empty") + .withDomains(List.of()) // Empty domains + .withDataProducts(List.of(dataProduct.getFullyQualifiedName())); + + // Should also fail during entity creation due to rule validation + assertResponse( + () -> tableResourceTest.createEntity(createTableEmpty, ADMIN_AUTH_HEADERS), + Response.Status.BAD_REQUEST, + DATA_PRODUCT_DOMAIN_RULE_EXC); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataProductDomainValidation_ComprehensiveScenarios(TestInfo test) throws IOException { + // Store original rule state for cleanup + SemanticsRule originalRule = getDataProductDomainRule(); + Boolean originalEnabled = originalRule.getEnabled(); + + try { + // Setup test data - domains and data products + Domain dataDomain = createTestDomain("Data", test); + Domain engineeringDomain = createTestDomain("Engineering", test); + Domain marketingDomain = createTestDomain("Marketing", test); + + DataProduct dataProductWithDataDomain = + createTestDataProduct("DataProduct", List.of(dataDomain), test); + DataProduct dataProductWithEngineeringDomain = + createTestDataProduct("EngineeringProduct", List.of(engineeringDomain), test); + DataProduct dataProductWithNoDomains = createTestDataProduct("NoDomainProduct", null, test); + + // Test 1: Matching domains - should pass + CreateTable matchingDomainsTable = + tableResourceTest + .createRequest(test.getDisplayName() + "_matching") + .withDomains(List.of(dataDomain.getFullyQualifiedName())) + .withDataProducts(List.of(dataProductWithDataDomain.getFullyQualifiedName())); + + Table createdTable = tableResourceTest.createEntity(matchingDomainsTable, ADMIN_AUTH_HEADERS); + assertNotNull(createdTable); + assertEquals(1, createdTable.getDomains().size()); + assertEquals(1, createdTable.getDataProducts().size()); + + // Test 2: Non-matching domains - should fail + CreateTable nonMatchingDomainsTable = + tableResourceTest + .createRequest(test.getDisplayName() + "_non_matching") + .withDomains(List.of(marketingDomain.getFullyQualifiedName())) + .withDataProducts(List.of(dataProductWithDataDomain.getFullyQualifiedName())); + + assertResponse( + () -> tableResourceTest.createEntity(nonMatchingDomainsTable, ADMIN_AUTH_HEADERS), + Response.Status.BAD_REQUEST, + DATA_PRODUCT_DOMAIN_RULE_EXC); + + // Test 3: Disable rule and retry - should pass + EntityResourceTest.toggleRule(DATA_PRODUCT_DOMAIN_RULE, false); + Table createdWithDisabledRule = + tableResourceTest.createEntity(nonMatchingDomainsTable, ADMIN_AUTH_HEADERS); + assertNotNull(createdWithDisabledRule); + + // Re-enable rule for remaining tests + EntityResourceTest.toggleRule(DATA_PRODUCT_DOMAIN_RULE, true); + + // Test 4: Entity with null domains but data products with domains - should fail + CreateTable nullDomainsTable = + tableResourceTest + .createRequest(test.getDisplayName() + "_null_domains") + .withDomains(null) + .withDataProducts(List.of(dataProductWithDataDomain.getFullyQualifiedName())); + + assertResponse( + () -> tableResourceTest.createEntity(nullDomainsTable, ADMIN_AUTH_HEADERS), + Response.Status.BAD_REQUEST, + DATA_PRODUCT_DOMAIN_RULE_EXC); + + // Test 5: Entity with domains but data products with null domains - should fail + CreateTable dataProductNullDomainsTable = + tableResourceTest + .createRequest(test.getDisplayName() + "_dp_null_domains") + .withDomains(List.of(dataDomain.getFullyQualifiedName())) + .withDataProducts(List.of(dataProductWithNoDomains.getFullyQualifiedName())); + + assertResponse( + () -> tableResourceTest.createEntity(dataProductNullDomainsTable, ADMIN_AUTH_HEADERS), + Response.Status.BAD_REQUEST, + DATA_PRODUCT_DOMAIN_RULE_EXC); + + // Test 6: No data products - should always pass + CreateTable noDataProductsTable = + tableResourceTest + .createRequest(test.getDisplayName() + "_no_data_products") + .withDomains(List.of(dataDomain.getFullyQualifiedName())) + .withDataProducts(null); + + Table tableWithNoDataProducts = + tableResourceTest.createEntity(noDataProductsTable, ADMIN_AUTH_HEADERS); + assertNotNull(tableWithNoDataProducts); + + // Test 7: Empty data products list - should also pass + CreateTable emptyDataProductsTable = + tableResourceTest + .createRequest(test.getDisplayName() + "_empty_data_products") + .withDomains(List.of(dataDomain.getFullyQualifiedName())) + .withDataProducts(List.of()); + + Table tableWithEmptyDataProducts = + tableResourceTest.createEntity(emptyDataProductsTable, ADMIN_AUTH_HEADERS); + assertNotNull(tableWithEmptyDataProducts); + + // Test 8: Disable rule for failing case and verify success + EntityResourceTest.toggleRule(DATA_PRODUCT_DOMAIN_RULE, false); + + CreateTable previouslyFailingTable = + tableResourceTest + .createRequest(test.getDisplayName() + "_previously_failing") + .withDomains(null) + .withDataProducts(List.of(dataProductWithDataDomain.getFullyQualifiedName())); + + Table successWithDisabledRule = + tableResourceTest.createEntity(previouslyFailingTable, ADMIN_AUTH_HEADERS); + assertNotNull(successWithDisabledRule); + + } finally { + // Restore original states + EntityResourceTest.toggleRule(DATA_PRODUCT_DOMAIN_RULE, originalEnabled); + } + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataProductDomainValidation_MultiDomainScenarios(TestInfo test) throws IOException { + // Store original states + SemanticsRule originalRule = getDataProductDomainRule(); + Boolean originalEnabled = originalRule.getEnabled(); + EntityResourceTest.toggleMultiDomainSupport(true); + + try { + // Setup test data + Domain dataDomain = createTestDomain("Data", test); + Domain engineeringDomain = createTestDomain("Engineering", test); + Domain marketingDomain = createTestDomain("Marketing", test); + + DataProduct matchingDataProduct = + createTestDataProduct("MatchingProduct", List.of(dataDomain), test); + DataProduct nonMatchingDataProduct = + createTestDataProduct("NonMatchingProduct", List.of(marketingDomain), test); + + // Test 1: Multiple domains with mixed matching data products - should fail + // First disable the multi-domain rule to test only the data product domain rule + EntityResourceTest.toggleRule("Multiple Domains are not allowed", false); + + CreateTable mixedMatchingTable = + tableResourceTest + .createRequest(test.getDisplayName() + "_mixed") + .withDomains( + List.of( + dataDomain.getFullyQualifiedName(), + engineeringDomain.getFullyQualifiedName())) + .withDataProducts( + List.of( + matchingDataProduct.getFullyQualifiedName(), + nonMatchingDataProduct.getFullyQualifiedName())); + + assertResponse( + () -> tableResourceTest.createEntity(mixedMatchingTable, ADMIN_AUTH_HEADERS), + Response.Status.BAD_REQUEST, + DATA_PRODUCT_DOMAIN_RULE_EXC); + + // Test 2: Disable rule and retry - should pass + EntityResourceTest.toggleRule(DATA_PRODUCT_DOMAIN_RULE, false); + Table createdWithDisabledRule = + tableResourceTest.createEntity(mixedMatchingTable, ADMIN_AUTH_HEADERS); + assertNotNull(createdWithDisabledRule); + assertEquals(2, createdWithDisabledRule.getDomains().size()); + assertEquals(2, createdWithDisabledRule.getDataProducts().size()); + + // Test 3: Re-enable rule and test with properly matching domains + EntityResourceTest.toggleRule(DATA_PRODUCT_DOMAIN_RULE, true); + + DataProduct anotherMatchingProduct = + createTestDataProduct("AnotherMatchingProduct", List.of(engineeringDomain), test); + + CreateTable properlyMatchingTable = + tableResourceTest + .createRequest(test.getDisplayName() + "_properly_matching") + .withDomains( + List.of( + dataDomain.getFullyQualifiedName(), + engineeringDomain.getFullyQualifiedName())) + .withDataProducts( + List.of( + matchingDataProduct.getFullyQualifiedName(), + anotherMatchingProduct.getFullyQualifiedName())); + + Table properlyCreatedTable = + tableResourceTest.createEntity(properlyMatchingTable, ADMIN_AUTH_HEADERS); + assertNotNull(properlyCreatedTable); + assertEquals(2, properlyCreatedTable.getDomains().size()); + assertEquals(2, properlyCreatedTable.getDataProducts().size()); + + } finally { + // Restore original states + EntityResourceTest.toggleRule(DATA_PRODUCT_DOMAIN_RULE, originalEnabled); + EntityResourceTest.toggleRule("Multiple Domains are not allowed", true); + EntityResourceTest.toggleMultiDomainSupport(false); + } + } }