mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-08-10 18:17:29 +00:00
MINOR - Don't enforce data contract on PATCH & cleanups (#22709)
* MINOR - Don't enforce data contract on PATCH & cleanups * fix pipeline refs for test suite * fix tests
This commit is contained in:
parent
bcc8cc1e39
commit
7407b3c9f5
@ -17,7 +17,6 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
|
|||||||
import static org.openmetadata.schema.type.EventType.ENTITY_CREATED;
|
import static org.openmetadata.schema.type.EventType.ENTITY_CREATED;
|
||||||
import static org.openmetadata.schema.type.EventType.ENTITY_UPDATED;
|
import static org.openmetadata.schema.type.EventType.ENTITY_UPDATED;
|
||||||
import static org.openmetadata.service.Entity.ADMIN_USER_NAME;
|
import static org.openmetadata.service.Entity.ADMIN_USER_NAME;
|
||||||
import static org.openmetadata.service.Entity.TEST_SUITE;
|
|
||||||
|
|
||||||
import jakarta.ws.rs.core.Response;
|
import jakarta.ws.rs.core.Response;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
@ -45,6 +44,7 @@ import org.openmetadata.schema.entity.datacontract.SchemaValidation;
|
|||||||
import org.openmetadata.schema.entity.datacontract.SemanticsValidation;
|
import org.openmetadata.schema.entity.datacontract.SemanticsValidation;
|
||||||
import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig;
|
import org.openmetadata.schema.entity.services.ingestionPipelines.AirflowConfig;
|
||||||
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
|
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
|
||||||
|
import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineServiceClientResponse;
|
||||||
import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType;
|
import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType;
|
||||||
import org.openmetadata.schema.metadataIngestion.SourceConfig;
|
import org.openmetadata.schema.metadataIngestion.SourceConfig;
|
||||||
import org.openmetadata.schema.metadataIngestion.TestSuitePipeline;
|
import org.openmetadata.schema.metadataIngestion.TestSuitePipeline;
|
||||||
@ -151,12 +151,56 @@ public class DataContractRepository extends EntityRepository<DataContract> {
|
|||||||
if (!nullOrEmpty(dataContract.getReviewers())) {
|
if (!nullOrEmpty(dataContract.getReviewers())) {
|
||||||
dataContract.setReviewers(EntityUtil.populateEntityReferences(dataContract.getReviewers()));
|
dataContract.setReviewers(EntityUtil.populateEntityReferences(dataContract.getReviewers()));
|
||||||
}
|
}
|
||||||
|
createOrUpdateDataContractTestSuite(dataContract);
|
||||||
|
}
|
||||||
|
|
||||||
TestSuite testSuite = createOrUpdateDataContractTestSuite(dataContract);
|
// Ensure we have a pipeline after creation if needed
|
||||||
// Create the ingestion pipeline only if needed
|
@Override
|
||||||
if (testSuite != null && nullOrEmpty(testSuite.getPipelines())) {
|
protected void postCreate(DataContract dataContract) {
|
||||||
IngestionPipeline pipeline = createIngestionPipeline(testSuite);
|
super.postCreate(dataContract);
|
||||||
prepareAndDeployIngestionPipeline(pipeline, testSuite);
|
postCreateOrUpdate(dataContract);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we update the contract adding DQ validation, add the pipeline if needed
|
||||||
|
@Override
|
||||||
|
protected void postUpdate(DataContract original, DataContract updated) {
|
||||||
|
super.postUpdate(original, updated);
|
||||||
|
postCreateOrUpdate(updated);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void postDelete(DataContract dataContract) {
|
||||||
|
super.postDelete(dataContract);
|
||||||
|
if (!nullOrEmpty(dataContract.getQualityExpectations())) {
|
||||||
|
TestSuite testSuite = getOrCreateTestSuite(dataContract);
|
||||||
|
TestSuiteRepository testSuiteRepository =
|
||||||
|
(TestSuiteRepository) Entity.getEntityRepository(Entity.TEST_SUITE);
|
||||||
|
testSuiteRepository.delete(ADMIN_USER_NAME, testSuite.getId(), true, true);
|
||||||
|
}
|
||||||
|
// Clean status
|
||||||
|
daoCollection
|
||||||
|
.entityExtensionTimeSeriesDao()
|
||||||
|
.delete(dataContract.getFullyQualifiedName(), RESULT_EXTENSION);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void postCreateOrUpdate(DataContract dataContract) {
|
||||||
|
if (!nullOrEmpty(dataContract.getQualityExpectations())) {
|
||||||
|
TestSuite testSuite = getOrCreateTestSuite(dataContract);
|
||||||
|
// Create the ingestion pipeline only if needed
|
||||||
|
if (testSuite != null && nullOrEmpty(testSuite.getPipelines())) {
|
||||||
|
IngestionPipeline pipeline = createIngestionPipeline(testSuite);
|
||||||
|
EntityReference pipelineRef =
|
||||||
|
Entity.getEntityReference(
|
||||||
|
new EntityReference().withId(pipeline.getId()).withType(Entity.INGESTION_PIPELINE),
|
||||||
|
Include.NON_DELETED);
|
||||||
|
testSuite.setPipelines(List.of(pipelineRef));
|
||||||
|
TestSuiteRepository testSuiteRepository =
|
||||||
|
(TestSuiteRepository) Entity.getEntityRepository(Entity.TEST_SUITE);
|
||||||
|
testSuiteRepository.createOrUpdate(null, testSuite, ADMIN_USER_NAME);
|
||||||
|
if (!pipeline.getDeployed()) {
|
||||||
|
prepareAndDeployIngestionPipeline(pipeline, testSuite);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -320,23 +364,12 @@ public class DataContractRepository extends EntityRepository<DataContract> {
|
|||||||
.withFullyQualifiedName(dataContract.getFullyQualifiedName())
|
.withFullyQualifiedName(dataContract.getFullyQualifiedName())
|
||||||
.withType(Entity.DATA_CONTRACT));
|
.withType(Entity.DATA_CONTRACT));
|
||||||
TestSuite newTestSuite = testSuiteMapper.createToEntity(createTestSuite, ADMIN_USER_NAME);
|
TestSuite newTestSuite = testSuiteMapper.createToEntity(createTestSuite, ADMIN_USER_NAME);
|
||||||
TestSuite createdSuite = testSuiteRepository.create(null, newTestSuite);
|
return testSuiteRepository.create(null, newTestSuite);
|
||||||
storeTestSuiteRelationship(dataContract, createdSuite);
|
|
||||||
return createdSuite;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return maybeTestSuite.get();
|
return maybeTestSuite.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void storeTestSuiteRelationship(DataContract dataContract, TestSuite testSuite) {
|
|
||||||
addRelationship(
|
|
||||||
dataContract.getId(),
|
|
||||||
testSuite.getId(),
|
|
||||||
Entity.DATA_CONTRACT,
|
|
||||||
TEST_SUITE,
|
|
||||||
Relationship.CONTAINS);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prepare the Ingestion Pipeline from the test suite that will handle the execution
|
// Prepare the Ingestion Pipeline from the test suite that will handle the execution
|
||||||
private IngestionPipeline createIngestionPipeline(TestSuite testSuite) {
|
private IngestionPipeline createIngestionPipeline(TestSuite testSuite) {
|
||||||
IngestionPipelineRepository pipelineRepository =
|
IngestionPipelineRepository pipelineRepository =
|
||||||
@ -414,7 +447,7 @@ public class DataContractRepository extends EntityRepository<DataContract> {
|
|||||||
// Otherwise, keep it Running and wait for the DQ results to kick in
|
// Otherwise, keep it Running and wait for the DQ results to kick in
|
||||||
if (!nullOrEmpty(dataContract.getQualityExpectations())) {
|
if (!nullOrEmpty(dataContract.getQualityExpectations())) {
|
||||||
try {
|
try {
|
||||||
triggerAndDeployDQValidation(dataContract);
|
deployAndTriggerDQValidation(dataContract);
|
||||||
compileResult(result, ContractExecutionStatus.Running);
|
compileResult(result, ContractExecutionStatus.Running);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
LOG.error(
|
LOG.error(
|
||||||
@ -435,7 +468,7 @@ public class DataContractRepository extends EntityRepository<DataContract> {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void triggerAndDeployDQValidation(DataContract dataContract) {
|
public void deployAndTriggerDQValidation(DataContract dataContract) {
|
||||||
if (dataContract.getTestSuite() == null) {
|
if (dataContract.getTestSuite() == null) {
|
||||||
throw DataContractValidationException.byMessage(
|
throw DataContractValidationException.byMessage(
|
||||||
String.format(
|
String.format(
|
||||||
@ -455,7 +488,11 @@ public class DataContractRepository extends EntityRepository<DataContract> {
|
|||||||
IngestionPipeline pipeline =
|
IngestionPipeline pipeline =
|
||||||
Entity.getEntity(testSuite.getPipelines().get(0), "*", Include.NON_DELETED);
|
Entity.getEntity(testSuite.getPipelines().get(0), "*", Include.NON_DELETED);
|
||||||
|
|
||||||
prepareAndDeployIngestionPipeline(pipeline, testSuite);
|
// ensure pipeline is deployed before running
|
||||||
|
// we deploy the pipeline during post create
|
||||||
|
if (!pipeline.getDeployed()) {
|
||||||
|
prepareAndDeployIngestionPipeline(pipeline, testSuite);
|
||||||
|
}
|
||||||
pipelineServiceClient.runPipeline(pipeline, testSuite);
|
pipelineServiceClient.runPipeline(pipeline, testSuite);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -466,7 +503,14 @@ public class DataContractRepository extends EntityRepository<DataContract> {
|
|||||||
SecretsManagerFactory.getSecretsManager()
|
SecretsManagerFactory.getSecretsManager()
|
||||||
.encryptOpenMetadataConnection(openMetadataServerConnection, false));
|
.encryptOpenMetadataConnection(openMetadataServerConnection, false));
|
||||||
|
|
||||||
pipelineServiceClient.deployPipeline(pipeline, testSuite);
|
PipelineServiceClientResponse response =
|
||||||
|
pipelineServiceClient.deployPipeline(pipeline, testSuite);
|
||||||
|
if (response.getCode() == 200) {
|
||||||
|
pipeline.setDeployed(true);
|
||||||
|
IngestionPipelineRepository ingestionPipelineRepository =
|
||||||
|
(IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE);
|
||||||
|
ingestionPipelineRepository.createOrUpdate(null, pipeline, ADMIN_USER_NAME);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private SemanticsValidation validateSemantics(DataContract dataContract) {
|
private SemanticsValidation validateSemantics(DataContract dataContract) {
|
||||||
@ -481,8 +525,11 @@ public class DataContractRepository extends EntityRepository<DataContract> {
|
|||||||
"*",
|
"*",
|
||||||
Include.NON_DELETED);
|
Include.NON_DELETED);
|
||||||
|
|
||||||
|
// We don't enforce the contract since we don't want to load it again. We're already passing
|
||||||
|
// its rules
|
||||||
List<SemanticsRule> failedRules =
|
List<SemanticsRule> failedRules =
|
||||||
RuleEngine.getInstance().evaluateAndReturn(entity, dataContract.getSemantics(), true);
|
RuleEngine.getInstance()
|
||||||
|
.evaluateAndReturn(entity, dataContract.getSemantics(), false, false);
|
||||||
|
|
||||||
validation
|
validation
|
||||||
.withFailed(failedRules.size())
|
.withFailed(failedRules.size())
|
||||||
|
@ -43,12 +43,16 @@ public class RuleEngine {
|
|||||||
* Evaluates the default platform entity semantics rules against the provided entity
|
* Evaluates the default platform entity semantics rules against the provided entity
|
||||||
*/
|
*/
|
||||||
public void evaluate(EntityInterface facts) {
|
public void evaluate(EntityInterface facts) {
|
||||||
evaluate(facts, null, false);
|
evaluate(facts, null, true, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void evaluate(EntityInterface facts, boolean enforcePlatform, boolean enforceContract) {
|
||||||
|
evaluate(facts, null, enforcePlatform, enforceContract);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void evaluateUpdate(EntityInterface original, EntityInterface updated) {
|
public void evaluateUpdate(EntityInterface original, EntityInterface updated) {
|
||||||
List<SemanticsRule> originalErrors = evaluateAndReturn(original, null, false);
|
List<SemanticsRule> originalErrors = evaluateAndReturn(original, null, true, false);
|
||||||
List<SemanticsRule> updatedErrors = evaluateAndReturn(updated, null, false);
|
List<SemanticsRule> updatedErrors = evaluateAndReturn(updated, null, true, false);
|
||||||
|
|
||||||
// If the updated entity is not fixing anything, throw a validation exception
|
// If the updated entity is not fixing anything, throw a validation exception
|
||||||
if (!nullOrEmpty(updatedErrors) && updatedErrors.size() >= originalErrors.size()) {
|
if (!nullOrEmpty(updatedErrors) && updatedErrors.size() >= originalErrors.size()) {
|
||||||
@ -56,12 +60,13 @@ public class RuleEngine {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void evaluate(EntityInterface facts, List<SemanticsRule> rules) {
|
public void evaluate(
|
||||||
evaluate(facts, rules, false);
|
EntityInterface facts,
|
||||||
}
|
List<SemanticsRule> rules,
|
||||||
|
boolean enforcePlatform,
|
||||||
public void evaluate(EntityInterface facts, List<SemanticsRule> rules, boolean incomingOnly) {
|
boolean enforceContract) {
|
||||||
List<SemanticsRule> erroredRules = evaluateAndReturn(facts, rules, incomingOnly);
|
List<SemanticsRule> erroredRules =
|
||||||
|
evaluateAndReturn(facts, rules, enforcePlatform, enforceContract);
|
||||||
raiseErroredRules(erroredRules);
|
raiseErroredRules(erroredRules);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -76,10 +81,37 @@ public class RuleEngine {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public List<SemanticsRule> evaluateAndReturn(
|
public List<SemanticsRule> evaluateAndReturn(
|
||||||
EntityInterface facts, List<SemanticsRule> rules, boolean incomingOnly) {
|
EntityInterface facts,
|
||||||
|
List<SemanticsRule> rules,
|
||||||
|
boolean enforcePlatform,
|
||||||
|
boolean enforceContract) {
|
||||||
|
List<SemanticsRule> rulesToEvaluate =
|
||||||
|
getRulesToEvaluate(facts, rules, enforcePlatform, enforceContract);
|
||||||
|
List<SemanticsRule> erroredRules = new ArrayList<>();
|
||||||
|
rulesToEvaluate.forEach(
|
||||||
|
rule -> {
|
||||||
|
if (shouldApplyRule(facts, rule)) {
|
||||||
|
try {
|
||||||
|
validateRule(facts, rule);
|
||||||
|
} catch (RuleValidationException e) {
|
||||||
|
erroredRules.add(rule);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return erroredRules;
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<SemanticsRule> getRulesToEvaluate(
|
||||||
|
EntityInterface facts,
|
||||||
|
List<SemanticsRule> rules,
|
||||||
|
boolean enforcePlatform,
|
||||||
|
boolean enforceContract) {
|
||||||
ArrayList<SemanticsRule> rulesToEvaluate = new ArrayList<>();
|
ArrayList<SemanticsRule> rulesToEvaluate = new ArrayList<>();
|
||||||
if (!incomingOnly) {
|
if (enforcePlatform) {
|
||||||
rulesToEvaluate.addAll(getEnabledEntitySemantics());
|
rulesToEvaluate.addAll(getEnabledEntitySemantics());
|
||||||
|
}
|
||||||
|
if (enforceContract) {
|
||||||
DataContract entityContract = dataContractRepository.getEntityDataContractSafely(facts);
|
DataContract entityContract = dataContractRepository.getEntityDataContractSafely(facts);
|
||||||
if (entityContract != null
|
if (entityContract != null
|
||||||
&& entityContract.getStatus() == ContractStatus.Active
|
&& entityContract.getStatus() == ContractStatus.Active
|
||||||
@ -94,20 +126,7 @@ public class RuleEngine {
|
|||||||
if (nullOrEmpty(rulesToEvaluate)) {
|
if (nullOrEmpty(rulesToEvaluate)) {
|
||||||
return List.of(); // No rules to evaluate
|
return List.of(); // No rules to evaluate
|
||||||
}
|
}
|
||||||
|
return rulesToEvaluate;
|
||||||
List<SemanticsRule> erroredRules = new ArrayList<>();
|
|
||||||
rulesToEvaluate.forEach(
|
|
||||||
rule -> {
|
|
||||||
if (shouldApplyRule(facts, rule)) {
|
|
||||||
try {
|
|
||||||
validateRule(facts, rule);
|
|
||||||
} catch (RuleValidationException e) {
|
|
||||||
erroredRules.add(rule);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
return erroredRules;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Boolean shouldApplyRule(EntityInterface facts, SemanticsRule rule) {
|
public Boolean shouldApplyRule(EntityInterface facts, SemanticsRule rule) {
|
||||||
|
@ -678,7 +678,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
|
|||||||
public abstract void assertFieldChange(String fieldName, Object expected, Object actual)
|
public abstract void assertFieldChange(String fieldName, Object expected, Object actual)
|
||||||
throws IOException;
|
throws IOException;
|
||||||
|
|
||||||
public void toggleMultiDomainSupport(Boolean enable) {
|
public static void toggleMultiDomainSupport(Boolean enable) {
|
||||||
SystemRepository systemRepository = Entity.getSystemRepository();
|
SystemRepository systemRepository = Entity.getSystemRepository();
|
||||||
|
|
||||||
Settings currentSettings =
|
Settings currentSettings =
|
||||||
|
@ -18,7 +18,9 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
import static org.openmetadata.service.resources.EntityResourceTest.TEAM11_REF;
|
import static org.openmetadata.service.resources.EntityResourceTest.TEAM11_REF;
|
||||||
import static org.openmetadata.service.resources.EntityResourceTest.USER1_REF;
|
import static org.openmetadata.service.resources.EntityResourceTest.USER1_REF;
|
||||||
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
|
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
|
||||||
@ -63,6 +65,7 @@ import org.openmetadata.schema.entity.data.Table;
|
|||||||
import org.openmetadata.schema.entity.datacontract.DataContractResult;
|
import org.openmetadata.schema.entity.datacontract.DataContractResult;
|
||||||
import org.openmetadata.schema.entity.services.DatabaseService;
|
import org.openmetadata.schema.entity.services.DatabaseService;
|
||||||
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
|
import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
|
||||||
|
import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineServiceClientResponse;
|
||||||
import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType;
|
import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineType;
|
||||||
import org.openmetadata.schema.entity.teams.User;
|
import org.openmetadata.schema.entity.teams.User;
|
||||||
import org.openmetadata.schema.services.connections.database.MysqlConnection;
|
import org.openmetadata.schema.services.connections.database.MysqlConnection;
|
||||||
@ -106,7 +109,6 @@ public class DataContractResourceTest extends OpenMetadataApplicationTest {
|
|||||||
private static TestCaseResourceTest testCaseResourceTest;
|
private static TestCaseResourceTest testCaseResourceTest;
|
||||||
private static IngestionPipelineResourceTest ingestionPipelineResourceTest;
|
private static IngestionPipelineResourceTest ingestionPipelineResourceTest;
|
||||||
private static DataContractRepository dataContractRepository;
|
private static DataContractRepository dataContractRepository;
|
||||||
private static PipelineServiceClientInterface originalPipelineClient;
|
|
||||||
private static PipelineServiceClientInterface mockPipelineClient;
|
private static PipelineServiceClientInterface mockPipelineClient;
|
||||||
|
|
||||||
@BeforeAll
|
@BeforeAll
|
||||||
@ -122,11 +124,22 @@ public class DataContractResourceTest extends OpenMetadataApplicationTest {
|
|||||||
org.openmetadata.service.Entity.getEntityRepository(
|
org.openmetadata.service.Entity.getEntityRepository(
|
||||||
org.openmetadata.service.Entity.DATA_CONTRACT);
|
org.openmetadata.service.Entity.DATA_CONTRACT);
|
||||||
|
|
||||||
// Store original client for potential restoration (if needed)
|
|
||||||
originalPipelineClient = dataContractRepository.getPipelineServiceClient();
|
|
||||||
|
|
||||||
// Create and set mock PipelineServiceClient
|
// Create and set mock PipelineServiceClient
|
||||||
mockPipelineClient = mock(PipelineServiceClientInterface.class);
|
mockPipelineClient = mock(PipelineServiceClientInterface.class);
|
||||||
|
|
||||||
|
// Configure mock to return successful response (code = 200) for all method calls
|
||||||
|
PipelineServiceClientResponse successResponse =
|
||||||
|
new PipelineServiceClientResponse()
|
||||||
|
.withCode(200)
|
||||||
|
.withReason("Success")
|
||||||
|
.withPlatform("test");
|
||||||
|
|
||||||
|
when(mockPipelineClient.deployPipeline(any(), any())).thenReturn(successResponse);
|
||||||
|
when(mockPipelineClient.runPipeline(any(), any())).thenReturn(successResponse);
|
||||||
|
when(mockPipelineClient.deletePipeline(any())).thenReturn(successResponse);
|
||||||
|
when(mockPipelineClient.toggleIngestion(any())).thenReturn(successResponse);
|
||||||
|
when(mockPipelineClient.killIngestion(any())).thenReturn(successResponse);
|
||||||
|
|
||||||
dataContractRepository.setPipelineServiceClient(mockPipelineClient);
|
dataContractRepository.setPipelineServiceClient(mockPipelineClient);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1910,6 +1923,9 @@ public class DataContractResourceTest extends OpenMetadataApplicationTest {
|
|||||||
assertEquals(testSuite.getId(), pipeline.getService().getId());
|
assertEquals(testSuite.getId(), pipeline.getService().getId());
|
||||||
assertEquals("testSuite", pipeline.getService().getType());
|
assertEquals("testSuite", pipeline.getService().getType());
|
||||||
|
|
||||||
|
// Also, the ingestion pipeline should be flagged as deployed
|
||||||
|
assertTrue(pipeline.getDeployed());
|
||||||
|
|
||||||
// Test deletion with recursive=true - should also delete the test suite
|
// Test deletion with recursive=true - should also delete the test suite
|
||||||
deleteDataContract(dataContract.getId(), true);
|
deleteDataContract(dataContract.getId(), true);
|
||||||
|
|
||||||
@ -1921,6 +1937,11 @@ public class DataContractResourceTest extends OpenMetadataApplicationTest {
|
|||||||
HttpResponseException.class,
|
HttpResponseException.class,
|
||||||
() ->
|
() ->
|
||||||
testSuiteResourceTest.getEntityByName(expectedTestSuiteName, "*", ADMIN_AUTH_HEADERS));
|
testSuiteResourceTest.getEntityByName(expectedTestSuiteName, "*", ADMIN_AUTH_HEADERS));
|
||||||
|
|
||||||
|
// Pipeline is also deleted
|
||||||
|
assertThrows(
|
||||||
|
HttpResponseException.class,
|
||||||
|
() -> ingestionPipelineResourceTest.getEntity(pipeline.getId(), "*", ADMIN_AUTH_HEADERS));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -36,7 +36,6 @@ import static org.openmetadata.service.exception.CatalogExceptionMessage.permiss
|
|||||||
import static org.openmetadata.service.jdbi3.RoleRepository.DOMAIN_ONLY_ACCESS_ROLE;
|
import static org.openmetadata.service.jdbi3.RoleRepository.DOMAIN_ONLY_ACCESS_ROLE;
|
||||||
import static org.openmetadata.service.resources.EntityResourceTest.C1;
|
import static org.openmetadata.service.resources.EntityResourceTest.C1;
|
||||||
import static org.openmetadata.service.resources.EntityResourceTest.DOMAIN_ONLY_ACCESS_ROLE_REF;
|
import static org.openmetadata.service.resources.EntityResourceTest.DOMAIN_ONLY_ACCESS_ROLE_REF;
|
||||||
import static org.openmetadata.service.resources.EntityResourceTest.MULTI_DOMAIN_RULE;
|
|
||||||
import static org.openmetadata.service.resources.EntityResourceTest.USER1;
|
import static org.openmetadata.service.resources.EntityResourceTest.USER1;
|
||||||
import static org.openmetadata.service.resources.EntityResourceTest.USER2_REF;
|
import static org.openmetadata.service.resources.EntityResourceTest.USER2_REF;
|
||||||
import static org.openmetadata.service.resources.EntityResourceTest.USER_ADDRESS_TAG_LABEL;
|
import static org.openmetadata.service.resources.EntityResourceTest.USER_ADDRESS_TAG_LABEL;
|
||||||
@ -91,15 +90,12 @@ import org.openmetadata.schema.api.feed.CreateThread;
|
|||||||
import org.openmetadata.schema.api.feed.ResolveTask;
|
import org.openmetadata.schema.api.feed.ResolveTask;
|
||||||
import org.openmetadata.schema.api.feed.ThreadCount;
|
import org.openmetadata.schema.api.feed.ThreadCount;
|
||||||
import org.openmetadata.schema.api.teams.CreateTeam;
|
import org.openmetadata.schema.api.teams.CreateTeam;
|
||||||
import org.openmetadata.schema.configuration.EntityRulesSettings;
|
|
||||||
import org.openmetadata.schema.entity.data.Table;
|
import org.openmetadata.schema.entity.data.Table;
|
||||||
import org.openmetadata.schema.entity.domains.Domain;
|
import org.openmetadata.schema.entity.domains.Domain;
|
||||||
import org.openmetadata.schema.entity.events.EventSubscription;
|
import org.openmetadata.schema.entity.events.EventSubscription;
|
||||||
import org.openmetadata.schema.entity.feed.Thread;
|
import org.openmetadata.schema.entity.feed.Thread;
|
||||||
import org.openmetadata.schema.entity.teams.Team;
|
import org.openmetadata.schema.entity.teams.Team;
|
||||||
import org.openmetadata.schema.entity.teams.User;
|
import org.openmetadata.schema.entity.teams.User;
|
||||||
import org.openmetadata.schema.settings.Settings;
|
|
||||||
import org.openmetadata.schema.settings.SettingsType;
|
|
||||||
import org.openmetadata.schema.type.AnnouncementDetails;
|
import org.openmetadata.schema.type.AnnouncementDetails;
|
||||||
import org.openmetadata.schema.type.ChatbotDetails;
|
import org.openmetadata.schema.type.ChatbotDetails;
|
||||||
import org.openmetadata.schema.type.Column;
|
import org.openmetadata.schema.type.Column;
|
||||||
@ -124,7 +120,7 @@ import org.openmetadata.service.formatter.decorators.MessageDecorator;
|
|||||||
import org.openmetadata.service.formatter.util.FeedMessage;
|
import org.openmetadata.service.formatter.util.FeedMessage;
|
||||||
import org.openmetadata.service.jdbi3.FeedRepository.FilterType;
|
import org.openmetadata.service.jdbi3.FeedRepository.FilterType;
|
||||||
import org.openmetadata.service.jdbi3.RoleRepository;
|
import org.openmetadata.service.jdbi3.RoleRepository;
|
||||||
import org.openmetadata.service.jdbi3.SystemRepository;
|
import org.openmetadata.service.resources.EntityResourceTest;
|
||||||
import org.openmetadata.service.resources.databases.TableResourceTest;
|
import org.openmetadata.service.resources.databases.TableResourceTest;
|
||||||
import org.openmetadata.service.resources.domains.DomainResourceTest;
|
import org.openmetadata.service.resources.domains.DomainResourceTest;
|
||||||
import org.openmetadata.service.resources.events.EventSubscriptionResourceTest;
|
import org.openmetadata.service.resources.events.EventSubscriptionResourceTest;
|
||||||
@ -1534,8 +1530,8 @@ public class FeedResourceTest extends OpenMetadataApplicationTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void list_threadsWithUserHavingMultipleDomains() throws HttpResponseException, IOException {
|
void list_threadsWithUserHavingMultipleDomains() throws IOException {
|
||||||
toggleMultiDomainSupport(false); // Disable multi-domain support rule for this test
|
EntityResourceTest.toggleMultiDomainSupport(false);
|
||||||
// Create domains for this test
|
// Create domains for this test
|
||||||
DomainResourceTest domainResourceTest = new DomainResourceTest();
|
DomainResourceTest domainResourceTest = new DomainResourceTest();
|
||||||
Domain testDomain1 =
|
Domain testDomain1 =
|
||||||
@ -1638,13 +1634,13 @@ public class FeedResourceTest extends OpenMetadataApplicationTest {
|
|||||||
// Clean up domains
|
// Clean up domains
|
||||||
domainResourceTest.deleteEntity(testDomain1.getId(), ADMIN_AUTH_HEADERS);
|
domainResourceTest.deleteEntity(testDomain1.getId(), ADMIN_AUTH_HEADERS);
|
||||||
domainResourceTest.deleteEntity(testDomain2.getId(), ADMIN_AUTH_HEADERS);
|
domainResourceTest.deleteEntity(testDomain2.getId(), ADMIN_AUTH_HEADERS);
|
||||||
|
EntityResourceTest.toggleMultiDomainSupport(true);
|
||||||
}
|
}
|
||||||
toggleMultiDomainSupport(true);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void list_threadsWithComplexDomainScenarios() throws HttpResponseException, IOException {
|
void list_threadsWithComplexDomainScenarios() throws IOException {
|
||||||
toggleMultiDomainSupport(false); // Disable multi-domain support rule for this test
|
EntityResourceTest.toggleMultiDomainSupport(false);
|
||||||
// Create test domains
|
// Create test domains
|
||||||
DomainResourceTest domainResourceTest = new DomainResourceTest();
|
DomainResourceTest domainResourceTest = new DomainResourceTest();
|
||||||
Domain testDomain1 =
|
Domain testDomain1 =
|
||||||
@ -1801,8 +1797,8 @@ public class FeedResourceTest extends OpenMetadataApplicationTest {
|
|||||||
domainResourceTest.deleteEntity(testDomain1.getId(), ADMIN_AUTH_HEADERS);
|
domainResourceTest.deleteEntity(testDomain1.getId(), ADMIN_AUTH_HEADERS);
|
||||||
domainResourceTest.deleteEntity(testDomain2.getId(), ADMIN_AUTH_HEADERS);
|
domainResourceTest.deleteEntity(testDomain2.getId(), ADMIN_AUTH_HEADERS);
|
||||||
domainResourceTest.deleteEntity(testDomain3.getId(), ADMIN_AUTH_HEADERS);
|
domainResourceTest.deleteEntity(testDomain3.getId(), ADMIN_AUTH_HEADERS);
|
||||||
|
EntityResourceTest.toggleMultiDomainSupport(true);
|
||||||
}
|
}
|
||||||
toggleMultiDomainSupport(true);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -2601,22 +2597,4 @@ public class FeedResourceTest extends OpenMetadataApplicationTest {
|
|||||||
public static String buildTableFieldLink(String tableFqn, String field) {
|
public static String buildTableFieldLink(String tableFqn, String field) {
|
||||||
return String.format("<#E::table::%s::%s>", tableFqn, field);
|
return String.format("<#E::table::%s::%s>", tableFqn, field);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void toggleMultiDomainSupport(Boolean enable) {
|
|
||||||
SystemRepository systemRepository = Entity.getSystemRepository();
|
|
||||||
|
|
||||||
Settings currentSettings =
|
|
||||||
systemRepository.getConfigWithKey(SettingsType.ENTITY_RULES_SETTINGS.toString());
|
|
||||||
EntityRulesSettings entityRulesSettings =
|
|
||||||
(EntityRulesSettings) currentSettings.getConfigValue();
|
|
||||||
entityRulesSettings
|
|
||||||
.getEntitySemantics()
|
|
||||||
.forEach(
|
|
||||||
rule -> {
|
|
||||||
if (MULTI_DOMAIN_RULE.equals(rule.getName())) {
|
|
||||||
rule.setEnabled(enable);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
systemRepository.updateSetting(currentSettings);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
@ -174,11 +174,11 @@ public class RuleEngineTests extends OpenMetadataApplicationTest {
|
|||||||
table.withOwners(List.of(USER1_REF));
|
table.withOwners(List.of(USER1_REF));
|
||||||
assertThrows(
|
assertThrows(
|
||||||
RuleValidationException.class,
|
RuleValidationException.class,
|
||||||
() -> RuleEngine.getInstance().evaluate(table, List.of(rule), true));
|
() -> RuleEngine.getInstance().evaluate(table, List.of(rule), false, false));
|
||||||
|
|
||||||
// Single team ownership should pass the Semantics Rule
|
// Single team ownership should pass the Semantics Rule
|
||||||
table.withOwners(List.of(TEAM11_REF));
|
table.withOwners(List.of(TEAM11_REF));
|
||||||
RuleEngine.getInstance().evaluate(table, List.of(rule), true);
|
RuleEngine.getInstance().evaluate(table, List.of(rule), false, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -202,7 +202,7 @@ public class RuleEngineTests extends OpenMetadataApplicationTest {
|
|||||||
"No glossary validation rule found for tables. Review the entityRulesSettings.json file."));
|
"No glossary validation rule found for tables. Review the entityRulesSettings.json file."));
|
||||||
|
|
||||||
// No glossary terms, should pass
|
// No glossary terms, should pass
|
||||||
RuleEngine.getInstance().evaluate(table, List.of(glossaryRule), true);
|
RuleEngine.getInstance().evaluate(table, List.of(glossaryRule), false, false);
|
||||||
|
|
||||||
// Single glossary term, should pass
|
// Single glossary term, should pass
|
||||||
table.withTags(
|
table.withTags(
|
||||||
@ -211,7 +211,7 @@ public class RuleEngineTests extends OpenMetadataApplicationTest {
|
|||||||
.withTagFQN("Glossary.Term1")
|
.withTagFQN("Glossary.Term1")
|
||||||
.withSource(TagLabel.TagSource.GLOSSARY)));
|
.withSource(TagLabel.TagSource.GLOSSARY)));
|
||||||
|
|
||||||
RuleEngine.getInstance().evaluate(table, List.of(glossaryRule), true);
|
RuleEngine.getInstance().evaluate(table, List.of(glossaryRule), false, false);
|
||||||
|
|
||||||
// Multiple glossary terms, should fail
|
// Multiple glossary terms, should fail
|
||||||
table.withTags(
|
table.withTags(
|
||||||
@ -224,7 +224,7 @@ public class RuleEngineTests extends OpenMetadataApplicationTest {
|
|||||||
.withSource(TagLabel.TagSource.GLOSSARY)));
|
.withSource(TagLabel.TagSource.GLOSSARY)));
|
||||||
assertThrows(
|
assertThrows(
|
||||||
RuleValidationException.class,
|
RuleValidationException.class,
|
||||||
() -> RuleEngine.getInstance().evaluate(table, List.of(glossaryRule), true));
|
() -> RuleEngine.getInstance().evaluate(table, List.of(glossaryRule), false, false));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -287,19 +287,21 @@ public class RuleEngineTests extends OpenMetadataApplicationTest {
|
|||||||
String tableJsonWithContract = JsonUtils.pojoToJson(table);
|
String tableJsonWithContract = JsonUtils.pojoToJson(table);
|
||||||
table.withOwners(List.of(TEAM11_REF));
|
table.withOwners(List.of(TEAM11_REF));
|
||||||
|
|
||||||
assertResponse(
|
// I can still PATCH the table to change the owners, even if the contract is broken.
|
||||||
() ->
|
// We don't have hard contract enforcement yet, so this is allowed.
|
||||||
tableResourceTest.patchEntity(
|
|
||||||
table.getId(), tableJsonWithContract, table, ADMIN_AUTH_HEADERS),
|
|
||||||
Response.Status.BAD_REQUEST,
|
|
||||||
"Rule [Description can't be empty] validation failed: Entity does not satisfy the rule. Rule context: Validates that the table has a description.");
|
|
||||||
|
|
||||||
// However, I can PATCH the table to add a proper description
|
|
||||||
table.withDescription("This is a valid description");
|
|
||||||
Table patched =
|
Table patched =
|
||||||
tableResourceTest.patchEntity(
|
tableResourceTest.patchEntity(
|
||||||
table.getId(), tableJsonWithContract, table, ADMIN_AUTH_HEADERS);
|
table.getId(), tableJsonWithContract, table, ADMIN_AUTH_HEADERS);
|
||||||
assertNotNull(patched);
|
assertNotNull(patched);
|
||||||
|
assertEquals(table.getOwners().getFirst().getId(), TEAM11_REF.getId());
|
||||||
|
|
||||||
|
// I can PATCH the table to add a proper description as well
|
||||||
|
tableJsonWithContract = JsonUtils.pojoToJson(patched);
|
||||||
|
table.withDescription("This is a valid description");
|
||||||
|
patched =
|
||||||
|
tableResourceTest.patchEntity(
|
||||||
|
table.getId(), tableJsonWithContract, table, ADMIN_AUTH_HEADERS);
|
||||||
|
assertNotNull(patched);
|
||||||
assertEquals("This is a valid description", patched.getDescription());
|
assertEquals("This is a valid description", patched.getDescription());
|
||||||
assertEquals(table.getOwners().getFirst().getId(), TEAM11_REF.getId());
|
assertEquals(table.getOwners().getFirst().getId(), TEAM11_REF.getId());
|
||||||
}
|
}
|
||||||
@ -350,7 +352,8 @@ public class RuleEngineTests extends OpenMetadataApplicationTest {
|
|||||||
|
|
||||||
// Table does indeed blow up
|
// Table does indeed blow up
|
||||||
assertThrows(
|
assertThrows(
|
||||||
RuleValidationException.class, () -> RuleEngine.getInstance().evaluate(tableWithContract));
|
RuleValidationException.class,
|
||||||
|
() -> RuleEngine.getInstance().evaluate(tableWithContract, false, true));
|
||||||
|
|
||||||
String tableJsonWithContract = JsonUtils.pojoToJson(tableWithContract);
|
String tableJsonWithContract = JsonUtils.pojoToJson(tableWithContract);
|
||||||
tableWithContract.withDescription("This is a valid description");
|
tableWithContract.withDescription("This is a valid description");
|
||||||
@ -364,7 +367,9 @@ public class RuleEngineTests extends OpenMetadataApplicationTest {
|
|||||||
ADMIN_AUTH_HEADERS);
|
ADMIN_AUTH_HEADERS);
|
||||||
|
|
||||||
// The patched table still blows up, since we've only fixed one rule
|
// The patched table still blows up, since we've only fixed one rule
|
||||||
assertThrows(RuleValidationException.class, () -> RuleEngine.getInstance().evaluate(patched));
|
assertThrows(
|
||||||
|
RuleValidationException.class,
|
||||||
|
() -> RuleEngine.getInstance().evaluate(patched, true, true));
|
||||||
|
|
||||||
String patchedJson = JsonUtils.pojoToJson(patched);
|
String patchedJson = JsonUtils.pojoToJson(patched);
|
||||||
patched.withOwners(List.of(TEAM11_REF));
|
patched.withOwners(List.of(TEAM11_REF));
|
||||||
@ -376,7 +381,7 @@ public class RuleEngineTests extends OpenMetadataApplicationTest {
|
|||||||
|
|
||||||
// Table is patched properly and is not blowing up anymore
|
// Table is patched properly and is not blowing up anymore
|
||||||
assertNotNull(fixedTable);
|
assertNotNull(fixedTable);
|
||||||
RuleEngine.getInstance().evaluate(fixedTable);
|
RuleEngine.getInstance().evaluate(fixedTable, true, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
Loading…
x
Reference in New Issue
Block a user