MINOR - Add Security, SLA and Terms of Use to Data Contract (#23230)

* jsonschema

* MINOR - Add Security, SLA and Terms of Use to Data Contract

* Update generated TypeScript types

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Pere Miquel Brull 2025-09-04 11:51:34 +02:00 committed by GitHub
parent 7961a583f5
commit 1e48241f51
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 480 additions and 2 deletions

View File

@ -82,9 +82,9 @@ import org.openmetadata.service.util.RestUtil;
public class DataContractRepository extends EntityRepository<DataContract> {
private static final String DATA_CONTRACT_UPDATE_FIELDS =
"entity,owners,reviewers,entityStatus,schema,qualityExpectations,contractUpdates,semantics,latestResult,extension";
"entity,owners,reviewers,entityStatus,schema,qualityExpectations,contractUpdates,semantics,termsOfUse,security,sla,latestResult,extension";
private static final String DATA_CONTRACT_PATCH_FIELDS =
"entity,owners,reviewers,entityStatus,schema,qualityExpectations,contractUpdates,semantics,latestResult,extension";
"entity,owners,reviewers,entityStatus,schema,qualityExpectations,contractUpdates,semantics,termsOfUse,security,sla,latestResult,extension";
public static final String RESULT_EXTENSION = "dataContract.dataContractResult";
public static final String RESULT_SCHEMA = "dataContractResult";
@ -877,6 +877,9 @@ public class DataContractRepository extends EntityRepository<DataContract> {
recordChange("latestResult", original.getLatestResult(), updated.getLatestResult());
recordChange("status", original.getEntityStatus(), updated.getEntityStatus());
recordChange("testSuite", original.getTestSuite(), updated.getTestSuite());
recordChange("termsOfUse", original.getTermsOfUse(), updated.getTermsOfUse());
recordChange("security", original.getSecurity(), updated.getSecurity());
recordChange("sla", original.getSla(), updated.getSla());
updateSchema(original, updated);
updateQualityExpectations(original, updated);
updateSemantics(original, updated);

View File

@ -43,6 +43,9 @@ public class DataContractMapper {
.withEffectiveFrom(create.getEffectiveFrom())
.withEffectiveUntil(create.getEffectiveUntil())
.withSourceUrl(create.getSourceUrl())
.withTermsOfUse(create.getTermsOfUse())
.withSecurity(create.getSecurity())
.withSla(create.getSla())
.withExtension(create.getExtension())
.withUpdatedBy(user)
.withUpdatedAt(System.currentTimeMillis());

View File

@ -4361,4 +4361,181 @@ public class DataContractResourceTest extends EntityResourceTest<DataContract, C
// Verify that the data contract is also deleted (should throw HttpResponseException)
assertThrows(HttpResponseException.class, () -> getDataContract(dataContract.getId(), null));
}
@Test
@Execution(ExecutionMode.CONCURRENT)
void testDataContractNewPropertiesFullLifecycle(TestInfo test) throws IOException {
// Test the full lifecycle of new properties: termsOfUse, security, and sla
Table table = createUniqueTable(test.getDisplayName());
// Create data contract with all new properties
String termsOfUse =
"# Terms of Use\n\nThis data is for internal use only.\n\n## Usage Guidelines\n- Do not share externally\n- Must comply with GDPR";
org.openmetadata.schema.api.data.ContractSecurity security =
new org.openmetadata.schema.api.data.ContractSecurity()
.withAccessPolicy("internal-only-policy")
.withDataClassification("Confidential");
org.openmetadata.schema.api.data.ContractSLA sla =
new org.openmetadata.schema.api.data.ContractSLA()
.withRefreshFrequency(
new org.openmetadata.schema.api.data.RefreshFrequency()
.withInterval(1)
.withUnit(org.openmetadata.schema.api.data.RefreshFrequency.Unit.DAY))
.withMaxLatency(
new org.openmetadata.schema.api.data.MaxLatency()
.withValue(4)
.withUnit(org.openmetadata.schema.api.data.MaxLatency.Unit.HOUR))
.withAvailabilityTime("09:00 UTC")
.withRetention(
new org.openmetadata.schema.api.data.Retention()
.withPeriod(90)
.withUnit(org.openmetadata.schema.api.data.Retention.Unit.DAY));
CreateDataContract create =
createDataContractRequest(test.getDisplayName(), table)
.withTermsOfUse(termsOfUse)
.withSecurity(security)
.withSla(sla);
// Test 1: Create data contract with new properties
DataContract created = createDataContract(create);
assertNotNull(created);
assertEquals(termsOfUse, created.getTermsOfUse());
assertNotNull(created.getSecurity());
assertEquals("internal-only-policy", created.getSecurity().getAccessPolicy());
assertEquals("Confidential", created.getSecurity().getDataClassification());
assertNotNull(created.getSla());
assertEquals(Integer.valueOf(1), created.getSla().getRefreshFrequency().getInterval());
assertEquals(
org.openmetadata.schema.api.data.RefreshFrequency.Unit.DAY,
created.getSla().getRefreshFrequency().getUnit());
assertEquals(Integer.valueOf(4), created.getSla().getMaxLatency().getValue());
assertEquals(
org.openmetadata.schema.api.data.MaxLatency.Unit.HOUR,
created.getSla().getMaxLatency().getUnit());
assertEquals("09:00 UTC", created.getSla().getAvailabilityTime());
assertEquals(Integer.valueOf(90), created.getSla().getRetention().getPeriod());
assertEquals(
org.openmetadata.schema.api.data.Retention.Unit.DAY,
created.getSla().getRetention().getUnit());
// Test 2: Read data contract and verify properties are retrieved
DataContract retrieved = getDataContract(created.getId(), null);
assertEquals(termsOfUse, retrieved.getTermsOfUse());
assertNotNull(retrieved.getSecurity());
assertEquals("internal-only-policy", retrieved.getSecurity().getAccessPolicy());
assertEquals("Confidential", retrieved.getSecurity().getDataClassification());
assertNotNull(retrieved.getSla());
assertEquals(Integer.valueOf(1), retrieved.getSla().getRefreshFrequency().getInterval());
assertEquals(
org.openmetadata.schema.api.data.RefreshFrequency.Unit.DAY,
retrieved.getSla().getRefreshFrequency().getUnit());
// Test 3: Update properties using PUT
String updatedTermsOfUse = "# Updated Terms\n\nNew terms apply from today.";
org.openmetadata.schema.api.data.ContractSecurity updatedSecurity =
new org.openmetadata.schema.api.data.ContractSecurity()
.withAccessPolicy("public-policy")
.withDataClassification("Public");
org.openmetadata.schema.api.data.ContractSLA updatedSla =
new org.openmetadata.schema.api.data.ContractSLA()
.withRefreshFrequency(
new org.openmetadata.schema.api.data.RefreshFrequency()
.withInterval(2)
.withUnit(org.openmetadata.schema.api.data.RefreshFrequency.Unit.HOUR))
.withMaxLatency(
new org.openmetadata.schema.api.data.MaxLatency()
.withValue(1)
.withUnit(org.openmetadata.schema.api.data.MaxLatency.Unit.HOUR))
.withAvailabilityTime("06:00 UTC");
create.withTermsOfUse(updatedTermsOfUse).withSecurity(updatedSecurity).withSla(updatedSla);
DataContract updated = updateDataContract(create);
assertEquals(updatedTermsOfUse, updated.getTermsOfUse());
assertEquals("public-policy", updated.getSecurity().getAccessPolicy());
assertEquals("Public", updated.getSecurity().getDataClassification());
assertEquals(Integer.valueOf(2), updated.getSla().getRefreshFrequency().getInterval());
assertEquals(
org.openmetadata.schema.api.data.RefreshFrequency.Unit.HOUR,
updated.getSla().getRefreshFrequency().getUnit());
assertEquals("06:00 UTC", updated.getSla().getAvailabilityTime());
assertNull(updated.getSla().getRetention()); // Verify retention was removed
// Test 4: Patch individual properties
String originalJson = JsonUtils.pojoToJson(updated);
// Patch only termsOfUse
String patchedTermsOfUse = "# Patched Terms\n\nOnly terms updated via patch.";
updated.setTermsOfUse(patchedTermsOfUse);
DataContract patched = patchDataContract(created.getId(), originalJson, updated);
assertEquals(patchedTermsOfUse, patched.getTermsOfUse());
// Verify other properties remain unchanged
assertEquals("public-policy", patched.getSecurity().getAccessPolicy());
assertEquals(Integer.valueOf(2), patched.getSla().getRefreshFrequency().getInterval());
// Test 5: Patch to remove properties (set to null)
originalJson = JsonUtils.pojoToJson(patched);
patched.setSecurity(null);
patched.setSla(null);
DataContract patchedWithNulls = patchDataContract(created.getId(), originalJson, patched);
assertEquals(patchedTermsOfUse, patchedWithNulls.getTermsOfUse());
assertNull(patchedWithNulls.getSecurity());
assertNull(patchedWithNulls.getSla());
// Test 6: Create contract with only termsOfUse (partial properties)
Table newTable = createUniqueTable(test.getDisplayName() + "_partial");
CreateDataContract partialCreate =
createDataContractRequest(test.getDisplayName() + "_partial", newTable)
.withTermsOfUse("Simple terms");
DataContract partial = createDataContract(partialCreate);
assertEquals("Simple terms", partial.getTermsOfUse());
assertNull(partial.getSecurity());
assertNull(partial.getSla());
// Test 7: Update to add security and sla to partial contract
partialCreate.withSecurity(security).withSla(sla);
DataContract partialUpdated = updateDataContract(partialCreate);
assertEquals("Simple terms", partialUpdated.getTermsOfUse());
assertNotNull(partialUpdated.getSecurity());
assertNotNull(partialUpdated.getSla());
// Test 8: Test with complex SLA configurations
org.openmetadata.schema.api.data.ContractSLA complexSla =
new org.openmetadata.schema.api.data.ContractSLA()
.withRefreshFrequency(
new org.openmetadata.schema.api.data.RefreshFrequency()
.withInterval(1)
.withUnit(org.openmetadata.schema.api.data.RefreshFrequency.Unit.MONTH))
.withMaxLatency(
new org.openmetadata.schema.api.data.MaxLatency()
.withValue(30)
.withUnit(org.openmetadata.schema.api.data.MaxLatency.Unit.MINUTE))
.withAvailabilityTime("23:59 UTC")
.withRetention(
new org.openmetadata.schema.api.data.Retention()
.withPeriod(7)
.withUnit(org.openmetadata.schema.api.data.Retention.Unit.YEAR));
Table complexTable = createUniqueTable(test.getDisplayName() + "_complex");
CreateDataContract complexCreate =
createDataContractRequest(test.getDisplayName() + "_complex", complexTable)
.withSla(complexSla);
DataContract complex = createDataContract(complexCreate);
assertNotNull(complex.getSla());
assertEquals(
org.openmetadata.schema.api.data.RefreshFrequency.Unit.MONTH,
complex.getSla().getRefreshFrequency().getUnit());
assertEquals(Integer.valueOf(30), complex.getSla().getMaxLatency().getValue());
assertEquals("23:59 UTC", complex.getSla().getAvailabilityTime());
assertEquals(Integer.valueOf(7), complex.getSla().getRetention().getPeriod());
assertEquals(
org.openmetadata.schema.api.data.Retention.Unit.YEAR,
complex.getSla().getRetention().getUnit());
}
}

View File

@ -77,6 +77,23 @@
"description": "Source URL of the data contract.",
"$ref": "../../type/basic.json#/definitions/sourceUrl"
},
"termsOfUse": {
"description": "Terms of use for the data contract for both human and AI agents consumption.",
"$ref": "../../type/basic.json#/definitions/markdown",
"default": null
},
"security": {
"title": "Contract Security",
"description": "Security and access policy expectations defined in the data contract.",
"$ref": "../../entity/data/dataContract.json#/definitions/contractSecurity",
"default": null
},
"sla": {
"title": "Contract SLA",
"description": "Service Level Agreement expectations defined in the data contract.",
"$ref": "../../entity/data/dataContract.json#/definitions/contractSLA",
"default": null
},
"extension": {
"description": "Entity extension data with custom attributes added to the entity.",
"$ref": "../../type/basic.json#/definitions/entityExtension"

View File

@ -59,6 +59,63 @@
"version"
],
"additionalProperties": false
},
"contractSecurity": {
"type": "object",
"description": "Security and access policy expectations",
"properties": {
"accessPolicy": {
"type": "string",
"title": "Access Policy",
"description": "Reference to an access policy ID or name that should govern this data"
},
"dataClassification": {
"type": "string",
"title": "Data Classification",
"description": "Expected data classification (e.g. Confidential, PII, etc.)"
}
}
},
"contractSLA": {
"type": "object",
"description": "Service Level Agreement expectations (timeliness, availability, etc.)",
"properties": {
"refreshFrequency": {
"type": "object",
"title": "Refresh Frequency",
"properties": {
"interval": { "type": "integer" },
"unit": { "type": "string", "enum": ["hour", "day", "week", "month", "year"] }
},
"description": "Expected frequency of data updates (e.g. every 1 day)",
"required": ["interval", "unit"]
},
"maxLatency": {
"type": "object",
"title": "Maximum Latency",
"properties": {
"value": { "type": "integer" },
"unit": { "type": "string", "enum": ["minute", "hour", "day"] }
},
"description": "Maximum acceptable latency between data generation and availability (e.g. 4 hours)",
"required": ["value", "unit"]
},
"availabilityTime": {
"title": "Availability Time",
"type": "string",
"description": "Time of day by which data is expected to be available (e.g. \"09:00 UTC\")"
},
"retention": {
"type": "object",
"title": "Data Retention Period",
"properties": {
"period": { "type": "integer" },
"unit": { "type": "string", "enum": ["day", "week", "month", "year"] }
},
"description": "How long the data is retained (if relevant)",
"required": ["period", "unit"]
}
}
}
},
"properties": {
@ -127,6 +184,23 @@
},
"default": null
},
"termsOfUse": {
"description": "Terms of use for the data contract for both human and AI agents consumption.",
"$ref": "../../type/basic.json#/definitions/markdown",
"default": null
},
"security": {
"title": "Contract Security",
"description": "Security and access policy expectations defined in the data contract.",
"$ref": "#/definitions/contractSecurity",
"default": null
},
"sla": {
"title": "Contract SLA",
"description": "Service Level Agreement expectations defined in the data contract.",
"$ref": "#/definitions/contractSLA",
"default": null
},
"qualityExpectations": {
"description": "Quality expectations defined in the data contract.",
"type": "array",

View File

@ -59,14 +59,26 @@ export interface CreateDataContract {
* Schema definition for the data contract.
*/
schema?: Column[];
/**
* Security and access policy expectations defined in the data contract.
*/
security?: ContractSecurity;
/**
* Semantics rules defined in the data contract.
*/
semantics?: SemanticsRule[];
/**
* Service Level Agreement expectations defined in the data contract.
*/
sla?: ContractSLA;
/**
* Source URL of the data contract.
*/
sourceUrl?: string;
/**
* Terms of use for the data contract for both human and AI agents consumption.
*/
termsOfUse?: string;
}
/**
@ -599,6 +611,23 @@ export interface Style {
iconURL?: string;
}
/**
* Security and access policy expectations defined in the data contract.
*
* Security and access policy expectations
*/
export interface ContractSecurity {
/**
* Reference to an access policy ID or name that should govern this data
*/
accessPolicy?: string;
/**
* Expected data classification (e.g. Confidential, PII, etc.)
*/
dataClassification?: string;
[property: string]: any;
}
/**
* Semantics rule defined in the data contract.
*/
@ -645,3 +674,76 @@ export enum ProviderType {
System = "system",
User = "user",
}
/**
* Service Level Agreement expectations defined in the data contract.
*
* Service Level Agreement expectations (timeliness, availability, etc.)
*/
export interface ContractSLA {
/**
* Time of day by which data is expected to be available (e.g. "09:00 UTC")
*/
availabilityTime?: string;
/**
* Maximum acceptable latency between data generation and availability (e.g. 4 hours)
*/
maxLatency?: MaximumLatency;
/**
* Expected frequency of data updates (e.g. every 1 day)
*/
refreshFrequency?: RefreshFrequency;
/**
* How long the data is retained (if relevant)
*/
retention?: DataRetentionPeriod;
[property: string]: any;
}
/**
* Maximum acceptable latency between data generation and availability (e.g. 4 hours)
*/
export interface MaximumLatency {
unit: MaxLatencyUnit;
value: number;
[property: string]: any;
}
export enum MaxLatencyUnit {
Day = "day",
Hour = "hour",
Minute = "minute",
}
/**
* Expected frequency of data updates (e.g. every 1 day)
*/
export interface RefreshFrequency {
interval: number;
unit: RefreshFrequencyUnit;
[property: string]: any;
}
export enum RefreshFrequencyUnit {
Day = "day",
Hour = "hour",
Month = "month",
Week = "week",
Year = "year",
}
/**
* How long the data is retained (if relevant)
*/
export interface DataRetentionPeriod {
period: number;
unit: RetentionUnit;
[property: string]: any;
}
export enum RetentionUnit {
Day = "day",
Month = "month",
Week = "week",
Year = "year",
}

View File

@ -94,14 +94,26 @@ export interface DataContract {
* Schema definition for the data contract.
*/
schema?: Column[];
/**
* Security and access policy expectations defined in the data contract.
*/
security?: ContractSecurity;
/**
* Semantics rules defined in the data contract.
*/
semantics?: SemanticsRule[];
/**
* Service Level Agreement expectations defined in the data contract.
*/
sla?: ContractSLA;
/**
* Source URL of the data contract.
*/
sourceUrl?: string;
/**
* Terms of use for the data contract for both human and AI agents consumption.
*/
termsOfUse?: string;
/**
* Reference to the test suite that contains tests related to this data contract.
*/
@ -766,6 +778,23 @@ export interface Style {
iconURL?: string;
}
/**
* Security and access policy expectations defined in the data contract.
*
* Security and access policy expectations
*/
export interface ContractSecurity {
/**
* Reference to an access policy ID or name that should govern this data
*/
accessPolicy?: string;
/**
* Expected data classification (e.g. Confidential, PII, etc.)
*/
dataClassification?: string;
[property: string]: any;
}
/**
* Semantics rule defined in the data contract.
*/
@ -812,3 +841,76 @@ export enum ProviderType {
System = "system",
User = "user",
}
/**
* Service Level Agreement expectations defined in the data contract.
*
* Service Level Agreement expectations (timeliness, availability, etc.)
*/
export interface ContractSLA {
/**
* Time of day by which data is expected to be available (e.g. "09:00 UTC")
*/
availabilityTime?: string;
/**
* Maximum acceptable latency between data generation and availability (e.g. 4 hours)
*/
maxLatency?: MaximumLatency;
/**
* Expected frequency of data updates (e.g. every 1 day)
*/
refreshFrequency?: RefreshFrequency;
/**
* How long the data is retained (if relevant)
*/
retention?: DataRetentionPeriod;
[property: string]: any;
}
/**
* Maximum acceptable latency between data generation and availability (e.g. 4 hours)
*/
export interface MaximumLatency {
unit: MaxLatencyUnit;
value: number;
[property: string]: any;
}
export enum MaxLatencyUnit {
Day = "day",
Hour = "hour",
Minute = "minute",
}
/**
* Expected frequency of data updates (e.g. every 1 day)
*/
export interface RefreshFrequency {
interval: number;
unit: RefreshFrequencyUnit;
[property: string]: any;
}
export enum RefreshFrequencyUnit {
Day = "day",
Hour = "hour",
Month = "month",
Week = "week",
Year = "year",
}
/**
* How long the data is retained (if relevant)
*/
export interface DataRetentionPeriod {
period: number;
unit: RetentionUnit;
[property: string]: any;
}
export enum RetentionUnit {
Day = "day",
Month = "month",
Week = "week",
Year = "year",
}