diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index 85eb74d30ee..3eb5fcaaf8b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -82,9 +82,9 @@ import org.openmetadata.service.util.RestUtil; public class DataContractRepository extends EntityRepository { private static final String DATA_CONTRACT_UPDATE_FIELDS = - "entity,owners,reviewers,entityStatus,schema,qualityExpectations,contractUpdates,semantics,latestResult,extension"; + "entity,owners,reviewers,entityStatus,schema,qualityExpectations,contractUpdates,semantics,termsOfUse,security,sla,latestResult,extension"; private static final String DATA_CONTRACT_PATCH_FIELDS = - "entity,owners,reviewers,entityStatus,schema,qualityExpectations,contractUpdates,semantics,latestResult,extension"; + "entity,owners,reviewers,entityStatus,schema,qualityExpectations,contractUpdates,semantics,termsOfUse,security,sla,latestResult,extension"; public static final String RESULT_EXTENSION = "dataContract.dataContractResult"; public static final String RESULT_SCHEMA = "dataContractResult"; @@ -877,6 +877,9 @@ public class DataContractRepository extends EntityRepository { recordChange("latestResult", original.getLatestResult(), updated.getLatestResult()); recordChange("status", original.getEntityStatus(), updated.getEntityStatus()); recordChange("testSuite", original.getTestSuite(), updated.getTestSuite()); + recordChange("termsOfUse", original.getTermsOfUse(), updated.getTermsOfUse()); + recordChange("security", original.getSecurity(), updated.getSecurity()); + recordChange("sla", original.getSla(), updated.getSla()); updateSchema(original, updated); updateQualityExpectations(original, updated); updateSemantics(original, updated); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractMapper.java index f59c2b51d44..846d285e068 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractMapper.java @@ -43,6 +43,9 @@ public class DataContractMapper { .withEffectiveFrom(create.getEffectiveFrom()) .withEffectiveUntil(create.getEffectiveUntil()) .withSourceUrl(create.getSourceUrl()) + .withTermsOfUse(create.getTermsOfUse()) + .withSecurity(create.getSecurity()) + .withSla(create.getSla()) .withExtension(create.getExtension()) .withUpdatedBy(user) .withUpdatedAt(System.currentTimeMillis()); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java index b7d710296a8..6de4d4a84b8 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/data/DataContractResourceTest.java @@ -4361,4 +4361,181 @@ public class DataContractResourceTest extends EntityResourceTest getDataContract(dataContract.getId(), null)); } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void testDataContractNewPropertiesFullLifecycle(TestInfo test) throws IOException { + // Test the full lifecycle of new properties: termsOfUse, security, and sla + Table table = createUniqueTable(test.getDisplayName()); + + // Create data contract with all new properties + String termsOfUse = + "# Terms of Use\n\nThis data is for internal use only.\n\n## Usage Guidelines\n- Do not share externally\n- Must comply with GDPR"; + + org.openmetadata.schema.api.data.ContractSecurity security = + new org.openmetadata.schema.api.data.ContractSecurity() + .withAccessPolicy("internal-only-policy") + .withDataClassification("Confidential"); + + org.openmetadata.schema.api.data.ContractSLA sla = + new org.openmetadata.schema.api.data.ContractSLA() + .withRefreshFrequency( + new org.openmetadata.schema.api.data.RefreshFrequency() + .withInterval(1) + .withUnit(org.openmetadata.schema.api.data.RefreshFrequency.Unit.DAY)) + .withMaxLatency( + new org.openmetadata.schema.api.data.MaxLatency() + .withValue(4) + .withUnit(org.openmetadata.schema.api.data.MaxLatency.Unit.HOUR)) + .withAvailabilityTime("09:00 UTC") + .withRetention( + new org.openmetadata.schema.api.data.Retention() + .withPeriod(90) + .withUnit(org.openmetadata.schema.api.data.Retention.Unit.DAY)); + + CreateDataContract create = + createDataContractRequest(test.getDisplayName(), table) + .withTermsOfUse(termsOfUse) + .withSecurity(security) + .withSla(sla); + + // Test 1: Create data contract with new properties + DataContract created = createDataContract(create); + assertNotNull(created); + assertEquals(termsOfUse, created.getTermsOfUse()); + assertNotNull(created.getSecurity()); + assertEquals("internal-only-policy", created.getSecurity().getAccessPolicy()); + assertEquals("Confidential", created.getSecurity().getDataClassification()); + assertNotNull(created.getSla()); + assertEquals(Integer.valueOf(1), created.getSla().getRefreshFrequency().getInterval()); + assertEquals( + org.openmetadata.schema.api.data.RefreshFrequency.Unit.DAY, + created.getSla().getRefreshFrequency().getUnit()); + assertEquals(Integer.valueOf(4), created.getSla().getMaxLatency().getValue()); + assertEquals( + org.openmetadata.schema.api.data.MaxLatency.Unit.HOUR, + created.getSla().getMaxLatency().getUnit()); + assertEquals("09:00 UTC", created.getSla().getAvailabilityTime()); + assertEquals(Integer.valueOf(90), created.getSla().getRetention().getPeriod()); + assertEquals( + org.openmetadata.schema.api.data.Retention.Unit.DAY, + created.getSla().getRetention().getUnit()); + + // Test 2: Read data contract and verify properties are retrieved + DataContract retrieved = getDataContract(created.getId(), null); + assertEquals(termsOfUse, retrieved.getTermsOfUse()); + assertNotNull(retrieved.getSecurity()); + assertEquals("internal-only-policy", retrieved.getSecurity().getAccessPolicy()); + assertEquals("Confidential", retrieved.getSecurity().getDataClassification()); + assertNotNull(retrieved.getSla()); + assertEquals(Integer.valueOf(1), retrieved.getSla().getRefreshFrequency().getInterval()); + assertEquals( + org.openmetadata.schema.api.data.RefreshFrequency.Unit.DAY, + retrieved.getSla().getRefreshFrequency().getUnit()); + + // Test 3: Update properties using PUT + String updatedTermsOfUse = "# Updated Terms\n\nNew terms apply from today."; + org.openmetadata.schema.api.data.ContractSecurity updatedSecurity = + new org.openmetadata.schema.api.data.ContractSecurity() + .withAccessPolicy("public-policy") + .withDataClassification("Public"); + + org.openmetadata.schema.api.data.ContractSLA updatedSla = + new org.openmetadata.schema.api.data.ContractSLA() + .withRefreshFrequency( + new org.openmetadata.schema.api.data.RefreshFrequency() + .withInterval(2) + .withUnit(org.openmetadata.schema.api.data.RefreshFrequency.Unit.HOUR)) + .withMaxLatency( + new org.openmetadata.schema.api.data.MaxLatency() + .withValue(1) + .withUnit(org.openmetadata.schema.api.data.MaxLatency.Unit.HOUR)) + .withAvailabilityTime("06:00 UTC"); + + create.withTermsOfUse(updatedTermsOfUse).withSecurity(updatedSecurity).withSla(updatedSla); + + DataContract updated = updateDataContract(create); + assertEquals(updatedTermsOfUse, updated.getTermsOfUse()); + assertEquals("public-policy", updated.getSecurity().getAccessPolicy()); + assertEquals("Public", updated.getSecurity().getDataClassification()); + assertEquals(Integer.valueOf(2), updated.getSla().getRefreshFrequency().getInterval()); + assertEquals( + org.openmetadata.schema.api.data.RefreshFrequency.Unit.HOUR, + updated.getSla().getRefreshFrequency().getUnit()); + assertEquals("06:00 UTC", updated.getSla().getAvailabilityTime()); + assertNull(updated.getSla().getRetention()); // Verify retention was removed + + // Test 4: Patch individual properties + String originalJson = JsonUtils.pojoToJson(updated); + + // Patch only termsOfUse + String patchedTermsOfUse = "# Patched Terms\n\nOnly terms updated via patch."; + updated.setTermsOfUse(patchedTermsOfUse); + DataContract patched = patchDataContract(created.getId(), originalJson, updated); + assertEquals(patchedTermsOfUse, patched.getTermsOfUse()); + // Verify other properties remain unchanged + assertEquals("public-policy", patched.getSecurity().getAccessPolicy()); + assertEquals(Integer.valueOf(2), patched.getSla().getRefreshFrequency().getInterval()); + + // Test 5: Patch to remove properties (set to null) + originalJson = JsonUtils.pojoToJson(patched); + patched.setSecurity(null); + patched.setSla(null); + DataContract patchedWithNulls = patchDataContract(created.getId(), originalJson, patched); + assertEquals(patchedTermsOfUse, patchedWithNulls.getTermsOfUse()); + assertNull(patchedWithNulls.getSecurity()); + assertNull(patchedWithNulls.getSla()); + + // Test 6: Create contract with only termsOfUse (partial properties) + Table newTable = createUniqueTable(test.getDisplayName() + "_partial"); + CreateDataContract partialCreate = + createDataContractRequest(test.getDisplayName() + "_partial", newTable) + .withTermsOfUse("Simple terms"); + + DataContract partial = createDataContract(partialCreate); + assertEquals("Simple terms", partial.getTermsOfUse()); + assertNull(partial.getSecurity()); + assertNull(partial.getSla()); + + // Test 7: Update to add security and sla to partial contract + partialCreate.withSecurity(security).withSla(sla); + DataContract partialUpdated = updateDataContract(partialCreate); + assertEquals("Simple terms", partialUpdated.getTermsOfUse()); + assertNotNull(partialUpdated.getSecurity()); + assertNotNull(partialUpdated.getSla()); + + // Test 8: Test with complex SLA configurations + org.openmetadata.schema.api.data.ContractSLA complexSla = + new org.openmetadata.schema.api.data.ContractSLA() + .withRefreshFrequency( + new org.openmetadata.schema.api.data.RefreshFrequency() + .withInterval(1) + .withUnit(org.openmetadata.schema.api.data.RefreshFrequency.Unit.MONTH)) + .withMaxLatency( + new org.openmetadata.schema.api.data.MaxLatency() + .withValue(30) + .withUnit(org.openmetadata.schema.api.data.MaxLatency.Unit.MINUTE)) + .withAvailabilityTime("23:59 UTC") + .withRetention( + new org.openmetadata.schema.api.data.Retention() + .withPeriod(7) + .withUnit(org.openmetadata.schema.api.data.Retention.Unit.YEAR)); + + Table complexTable = createUniqueTable(test.getDisplayName() + "_complex"); + CreateDataContract complexCreate = + createDataContractRequest(test.getDisplayName() + "_complex", complexTable) + .withSla(complexSla); + + DataContract complex = createDataContract(complexCreate); + assertNotNull(complex.getSla()); + assertEquals( + org.openmetadata.schema.api.data.RefreshFrequency.Unit.MONTH, + complex.getSla().getRefreshFrequency().getUnit()); + assertEquals(Integer.valueOf(30), complex.getSla().getMaxLatency().getValue()); + assertEquals("23:59 UTC", complex.getSla().getAvailabilityTime()); + assertEquals(Integer.valueOf(7), complex.getSla().getRetention().getPeriod()); + assertEquals( + org.openmetadata.schema.api.data.Retention.Unit.YEAR, + complex.getSla().getRetention().getUnit()); + } } diff --git a/openmetadata-spec/src/main/resources/json/schema/api/data/createDataContract.json b/openmetadata-spec/src/main/resources/json/schema/api/data/createDataContract.json index 21f32516363..56181bbb402 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/data/createDataContract.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/data/createDataContract.json @@ -77,6 +77,23 @@ "description": "Source URL of the data contract.", "$ref": "../../type/basic.json#/definitions/sourceUrl" }, + "termsOfUse": { + "description": "Terms of use for the data contract for both human and AI agents consumption.", + "$ref": "../../type/basic.json#/definitions/markdown", + "default": null + }, + "security": { + "title": "Contract Security", + "description": "Security and access policy expectations defined in the data contract.", + "$ref": "../../entity/data/dataContract.json#/definitions/contractSecurity", + "default": null + }, + "sla": { + "title": "Contract SLA", + "description": "Service Level Agreement expectations defined in the data contract.", + "$ref": "../../entity/data/dataContract.json#/definitions/contractSLA", + "default": null + }, "extension": { "description": "Entity extension data with custom attributes added to the entity.", "$ref": "../../type/basic.json#/definitions/entityExtension" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/dataContract.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/dataContract.json index 200946541e0..b5b833c9a1d 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/dataContract.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/dataContract.json @@ -59,6 +59,63 @@ "version" ], "additionalProperties": false + }, + "contractSecurity": { + "type": "object", + "description": "Security and access policy expectations", + "properties": { + "accessPolicy": { + "type": "string", + "title": "Access Policy", + "description": "Reference to an access policy ID or name that should govern this data" + }, + "dataClassification": { + "type": "string", + "title": "Data Classification", + "description": "Expected data classification (e.g. Confidential, PII, etc.)" + } + } + }, + "contractSLA": { + "type": "object", + "description": "Service Level Agreement expectations (timeliness, availability, etc.)", + "properties": { + "refreshFrequency": { + "type": "object", + "title": "Refresh Frequency", + "properties": { + "interval": { "type": "integer" }, + "unit": { "type": "string", "enum": ["hour", "day", "week", "month", "year"] } + }, + "description": "Expected frequency of data updates (e.g. every 1 day)", + "required": ["interval", "unit"] + }, + "maxLatency": { + "type": "object", + "title": "Maximum Latency", + "properties": { + "value": { "type": "integer" }, + "unit": { "type": "string", "enum": ["minute", "hour", "day"] } + }, + "description": "Maximum acceptable latency between data generation and availability (e.g. 4 hours)", + "required": ["value", "unit"] + }, + "availabilityTime": { + "title": "Availability Time", + "type": "string", + "description": "Time of day by which data is expected to be available (e.g. \"09:00 UTC\")" + }, + "retention": { + "type": "object", + "title": "Data Retention Period", + "properties": { + "period": { "type": "integer" }, + "unit": { "type": "string", "enum": ["day", "week", "month", "year"] } + }, + "description": "How long the data is retained (if relevant)", + "required": ["period", "unit"] + } + } } }, "properties": { @@ -127,6 +184,23 @@ }, "default": null }, + "termsOfUse": { + "description": "Terms of use for the data contract for both human and AI agents consumption.", + "$ref": "../../type/basic.json#/definitions/markdown", + "default": null + }, + "security": { + "title": "Contract Security", + "description": "Security and access policy expectations defined in the data contract.", + "$ref": "#/definitions/contractSecurity", + "default": null + }, + "sla": { + "title": "Contract SLA", + "description": "Service Level Agreement expectations defined in the data contract.", + "$ref": "#/definitions/contractSLA", + "default": null + }, "qualityExpectations": { "description": "Quality expectations defined in the data contract.", "type": "array", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDataContract.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDataContract.ts index 13ecaae5762..7c0278e5b95 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDataContract.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDataContract.ts @@ -59,14 +59,26 @@ export interface CreateDataContract { * Schema definition for the data contract. */ schema?: Column[]; + /** + * Security and access policy expectations defined in the data contract. + */ + security?: ContractSecurity; /** * Semantics rules defined in the data contract. */ semantics?: SemanticsRule[]; + /** + * Service Level Agreement expectations defined in the data contract. + */ + sla?: ContractSLA; /** * Source URL of the data contract. */ sourceUrl?: string; + /** + * Terms of use for the data contract for both human and AI agents consumption. + */ + termsOfUse?: string; } /** @@ -599,6 +611,23 @@ export interface Style { iconURL?: string; } +/** + * Security and access policy expectations defined in the data contract. + * + * Security and access policy expectations + */ +export interface ContractSecurity { + /** + * Reference to an access policy ID or name that should govern this data + */ + accessPolicy?: string; + /** + * Expected data classification (e.g. Confidential, PII, etc.) + */ + dataClassification?: string; + [property: string]: any; +} + /** * Semantics rule defined in the data contract. */ @@ -645,3 +674,76 @@ export enum ProviderType { System = "system", User = "user", } + +/** + * Service Level Agreement expectations defined in the data contract. + * + * Service Level Agreement expectations (timeliness, availability, etc.) + */ +export interface ContractSLA { + /** + * Time of day by which data is expected to be available (e.g. "09:00 UTC") + */ + availabilityTime?: string; + /** + * Maximum acceptable latency between data generation and availability (e.g. 4 hours) + */ + maxLatency?: MaximumLatency; + /** + * Expected frequency of data updates (e.g. every 1 day) + */ + refreshFrequency?: RefreshFrequency; + /** + * How long the data is retained (if relevant) + */ + retention?: DataRetentionPeriod; + [property: string]: any; +} + +/** + * Maximum acceptable latency between data generation and availability (e.g. 4 hours) + */ +export interface MaximumLatency { + unit: MaxLatencyUnit; + value: number; + [property: string]: any; +} + +export enum MaxLatencyUnit { + Day = "day", + Hour = "hour", + Minute = "minute", +} + +/** + * Expected frequency of data updates (e.g. every 1 day) + */ +export interface RefreshFrequency { + interval: number; + unit: RefreshFrequencyUnit; + [property: string]: any; +} + +export enum RefreshFrequencyUnit { + Day = "day", + Hour = "hour", + Month = "month", + Week = "week", + Year = "year", +} + +/** + * How long the data is retained (if relevant) + */ +export interface DataRetentionPeriod { + period: number; + unit: RetentionUnit; + [property: string]: any; +} + +export enum RetentionUnit { + Day = "day", + Month = "month", + Week = "week", + Year = "year", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dataContract.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dataContract.ts index f354ef8c046..372ee430429 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dataContract.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dataContract.ts @@ -94,14 +94,26 @@ export interface DataContract { * Schema definition for the data contract. */ schema?: Column[]; + /** + * Security and access policy expectations defined in the data contract. + */ + security?: ContractSecurity; /** * Semantics rules defined in the data contract. */ semantics?: SemanticsRule[]; + /** + * Service Level Agreement expectations defined in the data contract. + */ + sla?: ContractSLA; /** * Source URL of the data contract. */ sourceUrl?: string; + /** + * Terms of use for the data contract for both human and AI agents consumption. + */ + termsOfUse?: string; /** * Reference to the test suite that contains tests related to this data contract. */ @@ -766,6 +778,23 @@ export interface Style { iconURL?: string; } +/** + * Security and access policy expectations defined in the data contract. + * + * Security and access policy expectations + */ +export interface ContractSecurity { + /** + * Reference to an access policy ID or name that should govern this data + */ + accessPolicy?: string; + /** + * Expected data classification (e.g. Confidential, PII, etc.) + */ + dataClassification?: string; + [property: string]: any; +} + /** * Semantics rule defined in the data contract. */ @@ -812,3 +841,76 @@ export enum ProviderType { System = "system", User = "user", } + +/** + * Service Level Agreement expectations defined in the data contract. + * + * Service Level Agreement expectations (timeliness, availability, etc.) + */ +export interface ContractSLA { + /** + * Time of day by which data is expected to be available (e.g. "09:00 UTC") + */ + availabilityTime?: string; + /** + * Maximum acceptable latency between data generation and availability (e.g. 4 hours) + */ + maxLatency?: MaximumLatency; + /** + * Expected frequency of data updates (e.g. every 1 day) + */ + refreshFrequency?: RefreshFrequency; + /** + * How long the data is retained (if relevant) + */ + retention?: DataRetentionPeriod; + [property: string]: any; +} + +/** + * Maximum acceptable latency between data generation and availability (e.g. 4 hours) + */ +export interface MaximumLatency { + unit: MaxLatencyUnit; + value: number; + [property: string]: any; +} + +export enum MaxLatencyUnit { + Day = "day", + Hour = "hour", + Minute = "minute", +} + +/** + * Expected frequency of data updates (e.g. every 1 day) + */ +export interface RefreshFrequency { + interval: number; + unit: RefreshFrequencyUnit; + [property: string]: any; +} + +export enum RefreshFrequencyUnit { + Day = "day", + Hour = "hour", + Month = "month", + Week = "week", + Year = "year", +} + +/** + * How long the data is retained (if relevant) + */ +export interface DataRetentionPeriod { + period: number; + unit: RetentionUnit; + [property: string]: any; +} + +export enum RetentionUnit { + Day = "day", + Month = "month", + Week = "week", + Year = "year", +}