diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java index 1669bf6f671..b1654d6b47f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnUtil.java @@ -1,10 +1,15 @@ package org.openmetadata.service.jdbi3; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.util.FullyQualifiedName; @@ -62,4 +67,15 @@ public final class ColumnUtil { throw new IllegalArgumentException(CatalogExceptionMessage.invalidColumnFQN(columnFQN)); } } + + public static Set getAllTags(Column column) { + Set tags = new HashSet<>(); + if (!listOrEmpty(column.getTags()).isEmpty()) { + tags.addAll(column.getTags().stream().map(TagLabel::getTagFQN).collect(Collectors.toSet())); + } + for (Column c : listOrEmpty(column.getChildren())) { + tags.addAll(getAllTags(c)); + } + return tags; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index bd9804c3d75..299c1d3c9f0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -226,7 +226,7 @@ public class TableRepository extends EntityRepository { } @Transaction - public Table getSampleData(UUID tableId) throws IOException { + public Table getSampleData(UUID tableId, boolean authorizePII) throws IOException { // Validate the request content Table table = dao.findEntityById(tableId); @@ -236,6 +236,10 @@ public class TableRepository extends EntityRepository
{ TableData.class); table.setSampleData(sampleData); setFieldsInternal(table, Fields.EMPTY_FIELDS); + + // Set the column tags. Will be used to mask the sample data + if (!authorizePII) getColumnTags(true, table.getColumns()); + return table; } @@ -509,7 +513,7 @@ public class TableRepository extends EntityRepository
{ } @Transaction - public Table getLatestTableProfile(String fqn) throws IOException { + public Table getLatestTableProfile(String fqn, boolean authorizePII) throws IOException { Table table = dao.findEntityByName(fqn); TableProfile tableProfile = JsonUtils.readValue( @@ -519,6 +523,10 @@ public class TableRepository extends EntityRepository
{ TableProfile.class); table.setProfile(tableProfile); setColumnProfile(table.getColumns()); + + // Set the column tags. Will be used to hide the data + if (!authorizePII) getColumnTags(true, table.getColumns()); + return table; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index cb1d45056fd..fb10530c691 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -72,7 +72,9 @@ import org.openmetadata.service.jdbi3.TableRepository; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.mask.PIIMasker; import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.ResourceContext; import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.ResultList; @@ -531,8 +533,12 @@ public class TableResource extends EntityResource { @Parameter(description = "Id of the table", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) throws IOException { OperationContext operationContext = new OperationContext(entityType, MetadataOperation.VIEW_SAMPLE_DATA); - authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); - return addHref(uriInfo, repository.getSampleData(id)); + ResourceContext resourceContext = getResourceContextById(id); + authorizer.authorize(securityContext, operationContext, resourceContext); + boolean authorizePII = authorizer.authorizePII(securityContext, resourceContext.getOwner()); + + Table maskedTable = PIIMasker.getSampleData(repository.getSampleData(id, authorizePII), authorizePII); + return addHref(uriInfo, maskedTable); } @DELETE @@ -647,8 +653,11 @@ public class TableResource extends EntityResource { String fqn) throws IOException { OperationContext operationContext = new OperationContext(entityType, MetadataOperation.VIEW_DATA_PROFILE); - authorizer.authorize(securityContext, operationContext, getResourceContextByName(fqn)); - return repository.getLatestTableProfile(fqn); + ResourceContext resourceContext = getResourceContextByName(fqn); + authorizer.authorize(securityContext, operationContext, resourceContext); + boolean authorizePII = authorizer.authorizePII(securityContext, resourceContext.getOwner()); + + return PIIMasker.getTableProfile(repository.getLatestTableProfile(fqn, authorizePII), authorizePII); } @GET diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index b8e8fa31d67..0fe06b8d622 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -11,7 +11,9 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; import java.io.IOException; import java.util.List; +import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; import javax.json.JsonPatch; import javax.validation.Valid; import javax.validation.constraints.Max; @@ -37,9 +39,11 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.api.tests.CreateLogicalTestCases; import org.openmetadata.schema.api.tests.CreateTestCase; +import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.tests.type.TestCaseResult; +import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.EntityHistory; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; @@ -49,8 +53,10 @@ import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.TestCaseRepository; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; +import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; import org.openmetadata.service.security.Authorizer; +import org.openmetadata.service.security.mask.PIIMasker; import org.openmetadata.service.security.policyevaluator.OperationContext; import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; import org.openmetadata.service.security.policyevaluator.TestCaseResourceContext; @@ -175,8 +181,45 @@ public class TestCaseResource extends EntityResource tests = + super.listInternal( + uriInfo, securityContext, fields, filter, limitParam, before, after, operationContext, resourceContext); + + List maskedTests = + tests.getData().stream() + .map( + testCase -> { + try { + EntityLink testCaseLink = MessageParser.EntityLink.parse(testCase.getEntityLink()); + Table table = + Entity.getEntityByName( + Entity.TABLE, testCaseLink.getEntityFQN(), "owner,tags", Include.NON_DELETED); + + // Ignore table tests + if (testCaseLink.getFieldName() == null) return testCase; + + Optional referencedColumn = + table.getColumns().stream() + .filter( + col -> testCaseLink.getFullyQualifiedFieldValue().equals(col.getFullyQualifiedName())) + .findFirst(); + + if (referencedColumn.isPresent()) { + Column col = referencedColumn.get(); + // We need the table owner to know if we can authorize the access + boolean authorizePII = authorizer.authorizePII(securityContext, table.getOwner()); + return PIIMasker.getTestCase(col, testCase, authorizePII); + } + return testCase; + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + + tests.setData(maskedTests); + return tests; } @GET diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/Authorizer.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/Authorizer.java index 0d659093110..a72f6477989 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/Authorizer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/Authorizer.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.util.List; import javax.ws.rs.core.SecurityContext; import org.jdbi.v3.core.Jdbi; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.ResourcePermission; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.security.policyevaluator.OperationContext; @@ -46,4 +47,7 @@ public interface Authorizer { boolean decryptSecret(SecurityContext securityContext); boolean shouldMaskPasswords(SecurityContext securityContext); + + /** Let the user view PII Sensitive data */ + boolean authorizePII(SecurityContext securityContext, EntityReference owner); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java index 8bbd2118b64..90a1c1e1287 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java @@ -21,6 +21,7 @@ import java.util.List; import javax.ws.rs.core.SecurityContext; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.core.Jdbi; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.ResourcePermission; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.security.policyevaluator.OperationContext; @@ -97,6 +98,13 @@ public class DefaultAuthorizer implements Authorizer { return !subjectContext.isBot(); } + /** In 1.2, evaluate policies here instead of just checking the subject */ + @Override + public boolean authorizePII(SecurityContext securityContext, EntityReference owner) { + SubjectContext subjectContext = getSubjectContext(securityContext); + return subjectContext.isAdmin() || subjectContext.isBot() || subjectContext.isOwner(owner); + } + public static SubjectContext getSubjectContext(SecurityContext securityContext) { if (securityContext == null || securityContext.getUserPrincipal() == null) { throw new AuthenticationException("No principal in security context"); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java index ecb0b9074a5..015c6366464 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java @@ -20,6 +20,7 @@ import javax.ws.rs.core.SecurityContext; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.core.Jdbi; import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Permission.Access; import org.openmetadata.schema.type.ResourcePermission; import org.openmetadata.service.Entity; @@ -108,4 +109,9 @@ public class NoopAuthorizer implements Authorizer { public boolean shouldMaskPasswords(SecurityContext securityContext) { return false; // Always show passwords } + + @Override + public boolean authorizePII(SecurityContext securityContext, EntityReference owner) { + return true; // Always show PII Sensitive data + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/mask/PIIMasker.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/mask/PIIMasker.java new file mode 100644 index 00000000000..e2d6c56501c --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/mask/PIIMasker.java @@ -0,0 +1,83 @@ +package org.openmetadata.service.security.mask; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.TableData; +import org.openmetadata.service.jdbi3.ColumnUtil; + +public class PIIMasker { + + public static final String SENSITIVE_PII_TAG = "PII.Sensitive"; + public static final String MASKED_VALUE = "********"; + public static final String MASKED_NAME = "[MASKED]"; + + public static Table getSampleData(Table table, boolean authorized) { + if (authorized) return table; + + TableData sampleData = table.getSampleData(); + // get the list of positions to be masked + List columnsPositionToBeMasked = + table.getColumns().stream() + .collect(Collectors.toMap(Function.identity(), c -> sampleData.getColumns().indexOf(c.getName()))) + .entrySet() + .stream() + .filter(entry -> hasPiiSensitiveTag(entry.getKey())) + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + + // Mask rows + sampleData.setRows( + sampleData.getRows().stream() + .map(r -> maskSampleDataRow(r, columnsPositionToBeMasked)) + .collect(Collectors.toList())); + + List sampleDataColumns = sampleData.getColumns(); + + // Flag column names as masked + columnsPositionToBeMasked.forEach( + position -> sampleDataColumns.set(position, flagMaskedName(sampleDataColumns.get(position)))); + + table.setSampleData(sampleData); + return table; + } + + public static Table getTableProfile(Table table, boolean authorized) { + if (authorized) return table; + for (Column column : table.getColumns()) { + if (hasPiiSensitiveTag(column)) { + column.setProfile(null); + column.setName(flagMaskedName(column.getName())); + } + } + return table; + } + + public static TestCase getTestCase(Column column, TestCase testCase, boolean authorized) { + if (authorized || !hasPiiSensitiveTag(column)) return testCase; + + testCase.setTestCaseResult(null); + testCase.setParameterValues(null); + testCase.setDescription(null); + testCase.setName(flagMaskedName(testCase.getName())); + + return testCase; + } + + private static boolean hasPiiSensitiveTag(Column column) { + return ColumnUtil.getAllTags(column).stream().anyMatch(SENSITIVE_PII_TAG::equals); + } + + private static List maskSampleDataRow(List row, List columnsPositionToBeMasked) { + columnsPositionToBeMasked.forEach(position -> row.set(position, MASKED_VALUE)); + return row; + } + + private static String flagMaskedName(String name) { + return String.format("%s %s", name, MASKED_NAME); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index b230568f61d..3dbb4b70aa9 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -20,6 +20,7 @@ import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.OK; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -43,6 +44,7 @@ import static org.openmetadata.service.exception.CatalogExceptionMessage.entityN import static org.openmetadata.service.exception.CatalogExceptionMessage.invalidColumnFQN; import static org.openmetadata.service.exception.CatalogExceptionMessage.permissionNotAllowed; import static org.openmetadata.service.security.SecurityUtil.authHeaders; +import static org.openmetadata.service.security.mask.PIIMasker.MASKED_VALUE; import static org.openmetadata.service.util.EntityUtil.fieldAdded; import static org.openmetadata.service.util.EntityUtil.fieldDeleted; import static org.openmetadata.service.util.EntityUtil.fieldUpdated; @@ -1759,6 +1761,59 @@ public class TableResourceTest extends EntityResourceTest { assertEquals("P30D", table.getRetentionPeriod()); } + @Test + void test_sensitivePIISampleData(TestInfo test) throws IOException { + // Create table with owner and a column tagged with PII.Sensitive + Table table = + createAndCheckEntity(createRequest(test).withOwner(USER_TEAM21.getEntityReference()), ADMIN_AUTH_HEADERS); + List columns = Arrays.asList(C1, C2, C3); + // Add 3 rows of sample data for 3 columns + List> rows = + Arrays.asList( + Arrays.asList("c1Value1", 1, true), + Arrays.asList("c1Value2", null, false), + Arrays.asList("c1Value3", 3, true)); + // add sample data + putSampleData(table, columns, rows, ADMIN_AUTH_HEADERS); + // assert values are not masked for the table owner + table = getSampleData(table.getId(), authHeaders(USER_TEAM21.getName())); + assertFalse( + table.getSampleData().getRows().stream() + .flatMap(List::stream) + .map(r -> r == null ? "" : r) + .map(Object::toString) + .anyMatch(MASKED_VALUE::equals)); + // assert values are masked when is not the table owner + table = getSampleData(table.getId(), authHeaders(USER1_REF.getName())); + assertEquals( + 3, + table.getSampleData().getRows().stream() + .flatMap(List::stream) + .map(r -> r == null ? "" : r) + .map(Object::toString) + .filter(MASKED_VALUE::equals) + .count()); + } + + @Test + void test_sensitivePIIColumnProfile(TestInfo test) throws IOException, ParseException { + // Create table with owner and a column tagged with PII.Sensitive + // C3 has the PII.Sensitive tag + Table table = createEntity(createRequest(test).withOwner(USER_TEAM21.getEntityReference()), ADMIN_AUTH_HEADERS); + Table table1 = createEntity(createRequest(test, 1).withOwner(USER_TEAM21.getEntityReference()), ADMIN_AUTH_HEADERS); + putTableProfile(table, table1, ADMIN_AUTH_HEADERS); + + // Owner can read the column profile of C3 + Table tableWithProfileFromOwner = + getLatestTableProfile(table.getFullyQualifiedName(), authHeaders(USER_TEAM21.getName())); + assertNotNull(tableWithProfileFromOwner.getColumns().get(2).getProfile()); + + // Non owners cannot read the column profile of C3 + Table tableWithProfileFromNotOwner = + getLatestTableProfile(table.getFullyQualifiedName(), authHeaders(USER1_REF.getName())); + assertNull(tableWithProfileFromNotOwner.getColumns().get(2).getProfile()); + } + void assertFields(List
tableList, String fieldsParam) { tableList.forEach(t -> assertFields(t, fieldsParam)); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index 79f8686cba1..92137a4e193 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -6,6 +6,8 @@ import static javax.ws.rs.core.Response.Status.FORBIDDEN; import static javax.ws.rs.core.Response.Status.NOT_FOUND; import static javax.ws.rs.core.Response.Status.OK; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.schema.type.MetadataOperation.EDIT_TESTS; import static org.openmetadata.service.Entity.ADMIN_USER_NAME; @@ -16,6 +18,7 @@ import static org.openmetadata.service.util.EntityUtil.fieldUpdated; import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; import static org.openmetadata.service.util.TestUtils.TEST_AUTH_HEADERS; import static org.openmetadata.service.util.TestUtils.TEST_USER_NAME; +import static org.openmetadata.service.util.TestUtils.assertListNotEmpty; import static org.openmetadata.service.util.TestUtils.assertListNotNull; import static org.openmetadata.service.util.TestUtils.assertListNull; import static org.openmetadata.service.util.TestUtils.assertResponse; @@ -38,6 +41,7 @@ import org.junit.jupiter.api.TestMethodOrder; import org.openmetadata.schema.api.data.CreateTable; import org.openmetadata.schema.api.tests.CreateTestCase; import org.openmetadata.schema.api.tests.CreateTestSuite; +import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestCaseParameterValue; import org.openmetadata.schema.tests.TestSuite; @@ -333,6 +337,50 @@ public class TestCaseResourceTest extends EntityResourceTest", sensitiveTable.getFullyQualifiedName(), C1); + + CreateTestCase create = createRequest(test); + create + .withEntityLink(sensitiveColumnLink) + .withTestSuite(TEST_SUITE1.getFullyQualifiedName()) + .withTestDefinition(TEST_DEFINITION3.getFullyQualifiedName()) + .withParameterValues(List.of(new TestCaseParameterValue().withValue("100").withName("missingCountValue"))); + createAndCheckEntity(create, ADMIN_AUTH_HEADERS); + + // Owner can see the results + ResultList testCases = + getTestCases(10, "*", sensitiveColumnLink, false, authHeaders(USER1_REF.getName())); + assertNotNull(testCases.getData().get(0).getDescription()); + assertListNotEmpty(testCases.getData().get(0).getParameterValues()); + + // Owner can see the results + ResultList maskedTestCases = + getTestCases(10, "*", sensitiveColumnLink, false, authHeaders(USER2_REF.getName())); + assertNull(maskedTestCases.getData().get(0).getDescription()); + assertEquals(maskedTestCases.getData().get(0).getParameterValues().size(), 0); + } + @Test @Order(1) void put_testCase_list_200(TestInfo test) throws IOException {