From cffecc550af658ab4dcf232c3609aa42e9a4bd0e Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 31 Aug 2022 00:17:03 -0700 Subject: [PATCH] Fixes #7072 - Remove life cycle policies (#7073) * Fixes #7072 - Remove life cycle policies * Remove policy sink * Remove OMetaPolicy Co-authored-by: Pere Miquel Brull --- .../catalog/jdbi3/PolicyRepository.java | 18 ++------- .../resources/policies/PolicyResource.java | 6 +-- .../security/policyevaluator/PolicyCache.java | 13 +----- .../policyevaluator/SubjectContext.java | 4 -- .../openmetadata/catalog/util/EntityUtil.java | 10 ----- .../json/data/policy/DataConsumerPolicy.json | 1 - .../json/data/policy/DataStewardPolicy.json | 1 - .../json/data/policy/OrganizationPolicy.json | 1 - .../json/data/policy/TeamOnlyPolicy.json | 1 - .../schema/api/policies/createPolicy.json | 5 +-- .../entity/policies/accessControl/rule.json | 2 +- .../json/schema/entity/policies/filters.json | 22 ---------- .../policies/lifecycle/deleteAction.json | 21 ---------- .../entity/policies/lifecycle/moveAction.json | 40 ------------------- .../entity/policies/lifecycle/rule.json | 39 ------------------ .../json/schema/entity/policies/policy.json | 28 +------------ .../permissions/PermissionsResourceTest.java | 9 ++--- .../policies/PolicyResourceTest.java | 28 +++++-------- .../resources/teams/TeamResourceTest.java | 6 +-- .../policyevaluator/SubjectContextTest.java | 4 +- ingestion-core/src/metadata/_version.py | 2 +- .../metadata/ingestion/models/ometa_policy.py | 23 ----------- .../metadata/ingestion/sink/metadata_rest.py | 32 --------------- .../schemas/api/policies/createPolicy.md | 1 - .../schemas/entity/policies/policy.md | 2 - .../AddPolicyPage/AddPolicyPage.tsx | 2 - .../src/pages/PoliciesPage/policies.mock.ts | 1 - .../ui/src/pages/RolesPage/Roles.mock.ts | 4 -- 28 files changed, 28 insertions(+), 298 deletions(-) delete mode 100644 catalog-rest-service/src/main/resources/json/schema/entity/policies/filters.json delete mode 100644 catalog-rest-service/src/main/resources/json/schema/entity/policies/lifecycle/deleteAction.json delete mode 100644 catalog-rest-service/src/main/resources/json/schema/entity/policies/lifecycle/moveAction.json delete mode 100644 catalog-rest-service/src/main/resources/json/schema/entity/policies/lifecycle/rule.json delete mode 100644 ingestion/src/metadata/ingestion/models/ometa_policy.py diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java index 262004274a9..a6d91d3cac9 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/PolicyRepository.java @@ -19,7 +19,6 @@ import static org.openmetadata.catalog.Entity.LOCATION; import static org.openmetadata.catalog.Entity.POLICY; import static org.openmetadata.catalog.util.EntityUtil.entityReferenceMatch; import static org.openmetadata.catalog.util.EntityUtil.getRuleField; -import static org.openmetadata.catalog.util.EntityUtil.resolveRules; import static org.openmetadata.catalog.util.EntityUtil.ruleMatch; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; @@ -41,7 +40,6 @@ import org.openmetadata.catalog.resources.policies.PolicyResource; import org.openmetadata.catalog.security.policyevaluator.CompiledRule; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.MetadataOperation; -import org.openmetadata.catalog.type.PolicyType; import org.openmetadata.catalog.type.Relationship; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.EntityUtil.Fields; @@ -142,12 +140,8 @@ public class PolicyRepository extends EntityRepository { } public void validateRules(Policy policy) throws IOException { - if (!policy.getPolicyType().equals(PolicyType.AccessControl)) { - return; - } - // Resolve JSON blobs into Rule object and perform schema based validation - List rules = EntityUtil.resolveRules(policy.getRules()); + List rules = policy.getRules(); if (listOrEmpty(rules).isEmpty()) { throw new IllegalArgumentException(CatalogExceptionMessage.EMPTY_RULES_IN_POLICY); } @@ -162,13 +156,13 @@ public class PolicyRepository extends EntityRepository { } public List getAccessControlPolicies() throws IOException { - EntityUtil.Fields fields = new EntityUtil.Fields(List.of("policyType", "rules", ENABLED)); + EntityUtil.Fields fields = new EntityUtil.Fields(List.of("rules", ENABLED)); ListFilter filter = new ListFilter(); List jsons = daoCollection.policyDAO().listAfter(filter, Integer.MAX_VALUE, ""); List policies = new ArrayList<>(jsons.size()); for (String json : jsons) { Policy policy = setFields(JsonUtils.readValue(json, Policy.class), fields); - if (!policy.getPolicyType().equals(PolicyType.AccessControl) && !Boolean.TRUE.equals(policy.getEnabled())) { + if (!Boolean.TRUE.equals(policy.getEnabled())) { continue; } policies.add(policy); @@ -191,13 +185,9 @@ public class PolicyRepository extends EntityRepository { @Override public void entitySpecificUpdate() throws IOException { - // Disallow changing policyType. - if (original.getPolicyType() != updated.getPolicyType()) { - throw new IllegalArgumentException(CatalogExceptionMessage.readOnlyAttribute(POLICY, "policyType")); - } recordChange(ENABLED, original.getEnabled(), updated.getEnabled()); updateLocation(original, updated); - updateRules(resolveRules(original.getRules()), resolveRules(updated.getRules())); + updateRules(original.getRules(), updated.getRules()); } private void updateLocation(Policy origPolicy, Policy updatedPolicy) throws IOException { diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/policies/PolicyResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/policies/PolicyResource.java index 42cc7447e6a..ff8c7e666b7 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/policies/PolicyResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/policies/PolicyResource.java @@ -425,11 +425,7 @@ public class PolicyResource extends EntityResource { } private Policy getPolicy(CreatePolicy create, String user) throws IOException { - Policy policy = - copy(new Policy(), create, user) - .withPolicyType(create.getPolicyType()) - .withRules(create.getRules()) - .withEnabled(create.getEnabled()); + Policy policy = copy(new Policy(), create, user).withRules(create.getRules()).withEnabled(create.getEnabled()); if (create.getLocation() != null) { policy = policy.withLocation(new EntityReference().withId(create.getLocation())); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/security/policyevaluator/PolicyCache.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/security/policyevaluator/PolicyCache.java index 2c902d102d8..570e99a5a44 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/security/policyevaluator/PolicyCache.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/security/policyevaluator/PolicyCache.java @@ -30,7 +30,6 @@ import org.openmetadata.catalog.entity.policies.accessControl.Rule; import org.openmetadata.catalog.exception.EntityNotFoundException; import org.openmetadata.catalog.jdbi3.EntityRepository; import org.openmetadata.catalog.util.EntityUtil.Fields; -import org.openmetadata.catalog.util.JsonUtils; /** Subject context used for Access Control Policies */ @Slf4j @@ -74,16 +73,8 @@ public class PolicyCache { protected List getRules(Policy policy) { List rules = new ArrayList<>(); - for (Object r : policy.getRules()) { - try { - Rule rule = - JsonUtils.readValue( - JsonUtils.getJsonStructure(r).toString(), - org.openmetadata.catalog.entity.policies.accessControl.Rule.class); - rules.add(new CompiledRule(rule)); - } catch (Exception e) { - LOG.warn("Failed to load a rule", e); - } + for (Rule r : policy.getRules()) { + rules.add(new CompiledRule(r)); } return rules; } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/security/policyevaluator/SubjectContext.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/security/policyevaluator/SubjectContext.java index 380710db375..72eaa64409b 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/security/policyevaluator/SubjectContext.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/security/policyevaluator/SubjectContext.java @@ -256,15 +256,11 @@ public class SubjectContext { * roles are visited one by one, followed by the policies in the parent teams. */ static class TeamPolicyIterator implements Iterator { - private final UUID teamId; private int iteratorIndex = 0; private final List> iterators = new ArrayList<>(); - private final List teamsVisited; /** Policy iterator for a team */ TeamPolicyIterator(UUID teamId, List teamsVisited) { - this.teamId = teamId; - this.teamsVisited = teamsVisited; Team team = SubjectCache.getInstance().getTeam(teamId); // If a team is already visited (because user can belong to multiple teams diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java index 3a4a60611cb..4bada9a6d6b 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/EntityUtil.java @@ -413,16 +413,6 @@ public final class EntityUtil { .withDeleted(from.getDeleted()); } - public static List resolveRules(List rules) throws IOException { - List resolvedRules = new ArrayList<>(); - for (Object ruleObject : rules) { - // Cast to access control policy Rule. - resolvedRules.add( - JsonUtils.readValueWithValidation(JsonUtils.getJsonStructure(ruleObject).toString(), Rule.class)); - } - return resolvedRules; - } - public static TagLabel getTagLabel(GlossaryTerm term) { return new TagLabel() .withTagFQN(term.getFullyQualifiedName()) diff --git a/catalog-rest-service/src/main/resources/json/data/policy/DataConsumerPolicy.json b/catalog-rest-service/src/main/resources/json/data/policy/DataConsumerPolicy.json index 8ee7773bb1b..d982650f866 100644 --- a/catalog-rest-service/src/main/resources/json/data/policy/DataConsumerPolicy.json +++ b/catalog-rest-service/src/main/resources/json/data/policy/DataConsumerPolicy.json @@ -3,7 +3,6 @@ "displayName": "Data Consumer Policy", "fullyQualifiedName": "DataConsumerPolicy", "description": "Policy for Data Consumer to perform operations on metadata entities", - "policyType": "AccessControl", "enabled": true, "rules": [ { diff --git a/catalog-rest-service/src/main/resources/json/data/policy/DataStewardPolicy.json b/catalog-rest-service/src/main/resources/json/data/policy/DataStewardPolicy.json index a9d754d7507..0000191418c 100644 --- a/catalog-rest-service/src/main/resources/json/data/policy/DataStewardPolicy.json +++ b/catalog-rest-service/src/main/resources/json/data/policy/DataStewardPolicy.json @@ -3,7 +3,6 @@ "displayName": "Data Steward Policy", "fullyQualifiedName": "DataStewardPolicy", "description": "Policy for Data Steward Role to perform operations on metadata entities", - "policyType": "AccessControl", "enabled": true, "rules": [ { diff --git a/catalog-rest-service/src/main/resources/json/data/policy/OrganizationPolicy.json b/catalog-rest-service/src/main/resources/json/data/policy/OrganizationPolicy.json index df91e37bc70..d337fcd7659 100644 --- a/catalog-rest-service/src/main/resources/json/data/policy/OrganizationPolicy.json +++ b/catalog-rest-service/src/main/resources/json/data/policy/OrganizationPolicy.json @@ -3,7 +3,6 @@ "displayName": "Organization Policy", "fullyQualifiedName": "OrganizationPolicy", "description": "Policy for all the users of an organization.", - "policyType": "AccessControl", "enabled": true, "rules": [ { diff --git a/catalog-rest-service/src/main/resources/json/data/policy/TeamOnlyPolicy.json b/catalog-rest-service/src/main/resources/json/data/policy/TeamOnlyPolicy.json index c7c6116a2bf..703f803022b 100644 --- a/catalog-rest-service/src/main/resources/json/data/policy/TeamOnlyPolicy.json +++ b/catalog-rest-service/src/main/resources/json/data/policy/TeamOnlyPolicy.json @@ -3,7 +3,6 @@ "displayName": "Team only access Policy", "fullyQualifiedName": "TeamOnlyPolicy", "description": "Policy when attached to a team allows only users with in the team hierarchy to access the resources.", - "policyType": "AccessControl", "enabled": true, "rules": [ { diff --git a/catalog-rest-service/src/main/resources/json/schema/api/policies/createPolicy.json b/catalog-rest-service/src/main/resources/json/schema/api/policies/createPolicy.json index 4de90583739..fc893de38b5 100644 --- a/catalog-rest-service/src/main/resources/json/schema/api/policies/createPolicy.json +++ b/catalog-rest-service/src/main/resources/json/schema/api/policies/createPolicy.json @@ -24,9 +24,6 @@ "description": "Owner of this Policy.", "$ref": "../../type/entityReference.json" }, - "policyType": { - "$ref": "../../entity/policies/policy.json#/definitions/policyType" - }, "rules": { "$ref": "../../entity/policies/policy.json#/definitions/rules" }, @@ -41,6 +38,6 @@ "default": null } }, - "required": ["name", "policyType", "rules"], + "required": ["name", "rules"], "additionalProperties": false } diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/policies/accessControl/rule.json b/catalog-rest-service/src/main/resources/json/schema/entity/policies/accessControl/rule.json index 7370cc2173b..d36176cad57 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/policies/accessControl/rule.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/policies/accessControl/rule.json @@ -1,7 +1,7 @@ { "$id": "https://open-metadata.org/schema/entity/policies/accessControl/rule.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "AccessControlRule", + "title": "Rule", "description": "Describes an Access Control Rule for OpenMetadata Metadata Operations. All non-null user (subject) and entity (object) attributes are evaluated with logical AND.", "type": "object", "javaType": "org.openmetadata.catalog.entity.policies.accessControl.Rule", diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/policies/filters.json b/catalog-rest-service/src/main/resources/json/schema/entity/policies/filters.json deleted file mode 100644 index 8da3ca2a829..00000000000 --- a/catalog-rest-service/src/main/resources/json/schema/entity/policies/filters.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/entity/policies/filters.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Filters", - "definitions": { - "prefix": { - "description": "Prefix path of the entity.", - "type": "string" - }, - "regex": { - "description": "Regex that matches the entity.", - "type": "string" - }, - "tags": { - "description": "Set of tags to match on (OR among all tags).", - "type": "array", - "items": { - "$ref": "../tags/tagCategory.json#/definitions/tagName" - } - } - } -} diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/policies/lifecycle/deleteAction.json b/catalog-rest-service/src/main/resources/json/schema/entity/policies/lifecycle/deleteAction.json deleted file mode 100644 index 039829c4931..00000000000 --- a/catalog-rest-service/src/main/resources/json/schema/entity/policies/lifecycle/deleteAction.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/entity/policies/lifecycle/deleteAction.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "LifecycleDeleteAction", - "description": "An action to delete or expire the entity.", - "type": "object", - "javaType": "org.openmetadata.catalog.entity.policies.lifecycle.DeleteAction", - "properties": { - "daysAfterCreation": { - "type": "integer", - "description": "Number of days after creation of the entity that the deletion should be triggered.", - "minimum": 1 - }, - "daysAfterModification": { - "type": "integer", - "description": "Number of days after last modification of the entity that the deletion should be triggered.", - "minimum": 1 - } - }, - "additionalProperties": false -} diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/policies/lifecycle/moveAction.json b/catalog-rest-service/src/main/resources/json/schema/entity/policies/lifecycle/moveAction.json deleted file mode 100644 index f5bb6a93234..00000000000 --- a/catalog-rest-service/src/main/resources/json/schema/entity/policies/lifecycle/moveAction.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/entity/policies/lifecycle/moveAction.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "LifecycleMoveAction", - "description": "An action to move the entity to a different location. For eg: Move from Standard storage tier to Archive storage tier.", - "type": "object", - "javaType": "org.openmetadata.catalog.entity.policies.lifecycle.MoveAction", - "properties": { - "daysAfterCreation": { - "description": "Number of days after creation of the entity that the move should be triggered.", - "type": "integer", - "minimum": 1 - }, - "daysAfterModification": { - "description": "Number of days after last modification of the entity that the move should be triggered.", - "type": "integer", - "minimum": 1 - }, - "destination": { - "description": "Location where this entity needs to be moved to.", - "type": "object", - "properties": { - "storageServiceType": { - "description": "The storage service to move this entity to.", - "$ref": "../../services/storageService.json" - }, - "storageClassType": { - "description": "The storage class to move this entity to.", - "$ref": "../../../type/storage.json#/definitions/storageClassType" - }, - "location": { - "description": "The location where to move this entity to.", - "$ref": "../../data/location.json" - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false -} diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/policies/lifecycle/rule.json b/catalog-rest-service/src/main/resources/json/schema/entity/policies/lifecycle/rule.json deleted file mode 100644 index ed3b9de175d..00000000000 --- a/catalog-rest-service/src/main/resources/json/schema/entity/policies/lifecycle/rule.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "$id": "https://open-metadata.org/schema/entity/policies/lifecycle/rule.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "LifecycleRule", - "description": "Describes an entity Lifecycle Rule used within a Policy.", - "type": "object", - "javaType": "org.openmetadata.catalog.entity.policies.lifecycle.Rule", - "properties": { - "name": { - "description": "Name that identifies this Rule.", - "type": "string" - }, - "prefixFilter": { - "$ref": "../filters.json#/definitions/prefix" - }, - "regexFilter": { - "$ref": "../filters.json#/definitions/regex" - }, - "tagsFilter": { - "$ref": "../filters.json#/definitions/tags" - }, - "actions": { - "description": "A set of actions to take on the entities.", - "type": "array", - "items": { - "anyOf": [ - { - "$ref": "deleteAction.json" - }, - { - "$ref": "moveAction.json" - } - ] - } - } - }, - "required": ["actions"], - "additionalProperties": false -} diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/policies/policy.json b/catalog-rest-service/src/main/resources/json/schema/entity/policies/policy.json index cd088d1803b..cd76efb87a7 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/policies/policy.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/policies/policy.json @@ -7,32 +7,11 @@ "javaType": "org.openmetadata.catalog.entity.policies.Policy", "javaInterfaces": ["org.openmetadata.catalog.EntityInterface"], "definitions": { - "policyType": { - "javaType": "org.openmetadata.catalog.type.PolicyType", - "description": "This schema defines the type used for describing different types of policies.", - "type": "string", - "enum": ["AccessControl", "Lifecycle"], - "javaEnums": [ - { - "name": "AccessControl" - }, - { - "name": "Lifecycle" - } - ] - }, "rules": { "description": "A set of rules associated with the Policy.", "type": "array", "items": { - "anyOf": [ - { - "$ref": "accessControl/rule.json" - }, - { - "$ref": "lifecycle/rule.json" - } - ] + "$ref": "accessControl/rule.json" } } }, @@ -66,9 +45,6 @@ "description": "Link to the resource corresponding to this entity.", "$ref": "../../type/basic.json#/definitions/href" }, - "policyType": { - "$ref": "#/definitions/policyType" - }, "enabled": { "description": "Is the policy enabled.", "type": "boolean", @@ -113,6 +89,6 @@ "default": false } }, - "required": ["id", "name", "policyType", "rules"], + "required": ["id", "name", "rules"], "additionalProperties": false } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/permissions/PermissionsResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/permissions/PermissionsResourceTest.java index c3270ddd714..3a6643e9994 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/permissions/PermissionsResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/permissions/PermissionsResourceTest.java @@ -68,7 +68,6 @@ import org.openmetadata.catalog.type.Permission; import org.openmetadata.catalog.type.Permission.Access; import org.openmetadata.catalog.type.ResourceDescriptor; import org.openmetadata.catalog.type.ResourcePermission; -import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.TestUtils; @Slf4j @@ -97,21 +96,21 @@ class PermissionsResourceTest extends CatalogApplicationTest { Policy ORG_POLICY = policyResourceTest.getEntityByName(ORG_POLICY_NAME, null, PolicyResource.FIELDS, ADMIN_AUTH_HEADERS); - ORG_RULES = EntityUtil.resolveRules(ORG_POLICY.getRules()); + ORG_RULES = ORG_POLICY.getRules(); Policy DATA_STEWARD_POLICY = policyResourceTest.getEntityByName(DATA_STEWARD_POLICY_NAME, null, PolicyResource.FIELDS, ADMIN_AUTH_HEADERS); - DATA_STEWARD_RULES = EntityUtil.resolveRules(DATA_STEWARD_POLICY.getRules()); + DATA_STEWARD_RULES = DATA_STEWARD_POLICY.getRules(); DATA_STEWARD_POLICY = policyResourceTest.getEntityByName(DATA_STEWARD_POLICY_NAME, null, PolicyResource.FIELDS, ADMIN_AUTH_HEADERS); - DATA_STEWARD_RULES = EntityUtil.resolveRules(DATA_STEWARD_POLICY.getRules()); + DATA_STEWARD_RULES = DATA_STEWARD_POLICY.getRules(); DATA_STEWARD_USER = EntityResourceTest.USER_WITH_DATA_STEWARD_ROLE; Policy DATA_CONSUMER_POLICY = policyResourceTest.getEntityByName(DATA_CONSUMER_POLICY_NAME, null, PolicyResource.FIELDS, ADMIN_AUTH_HEADERS); - DATA_CONSUMER_RULES = EntityUtil.resolveRules(DATA_CONSUMER_POLICY.getRules()); + DATA_CONSUMER_RULES = DATA_CONSUMER_POLICY.getRules(); DATA_CONSUMER_USER = EntityResourceTest.USER_WITH_DATA_CONSUMER_ROLE; } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java index 33ac2662ab4..8c9df3282cd 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java @@ -25,7 +25,6 @@ import static org.openmetadata.catalog.util.EntityUtil.fieldAdded; import static org.openmetadata.catalog.util.EntityUtil.fieldDeleted; import static org.openmetadata.catalog.util.EntityUtil.fieldUpdated; import static org.openmetadata.catalog.util.EntityUtil.getRuleField; -import static org.openmetadata.catalog.util.EntityUtil.resolveRules; import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS; import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE; import static org.openmetadata.catalog.util.TestUtils.assertListNotNull; @@ -37,10 +36,10 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Random; -import java.util.stream.Collectors; import javax.ws.rs.client.WebTarget; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -72,7 +71,6 @@ import org.openmetadata.catalog.type.ChangeDescription; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.Function; import org.openmetadata.catalog.type.MetadataOperation; -import org.openmetadata.catalog.type.PolicyType; import org.openmetadata.catalog.type.ResourceDescriptor; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.JsonUtils; @@ -98,7 +96,7 @@ public class PolicyResourceTest extends EntityResourceTest POLICY1 = createEntity(createRequest("policy1").withOwner(null), ADMIN_AUTH_HEADERS); POLICY2 = createEntity(createRequest("policy2").withOwner(null), ADMIN_AUTH_HEADERS); TEAM_ONLY_POLICY = getEntityByName("TeamOnlyPolicy", "", ADMIN_AUTH_HEADERS); - TEAM_ONLY_POLICY_RULES = EntityUtil.resolveRules(TEAM_ONLY_POLICY.getRules()); + TEAM_ONLY_POLICY_RULES = TEAM_ONLY_POLICY.getRules(); } @Override @@ -111,11 +109,14 @@ public class PolicyResourceTest extends EntityResourceTest @Override @SneakyThrows public void validateCreatedEntity(Policy policy, CreatePolicy createRequest, Map authHeaders) { - assertEquals(createRequest.getPolicyType(), policy.getPolicyType()); if (createRequest.getLocation() != null) { assertEquals(createRequest.getLocation(), policy.getLocation().getId()); } - assertEquals(createRequest.getRules(), resolveRules(policy.getRules())); + if (createRequest.getRules().size() > 1) { + createRequest.getRules().sort(Comparator.comparing(Rule::getName)); + } + policy.getRules().sort(Comparator.comparing(Rule::getName)); + assertEquals(createRequest.getRules(), policy.getRules()); } @Override @@ -136,7 +137,7 @@ public class PolicyResourceTest extends EntityResourceTest assertEquals(expectedLocation.getId(), actualLocation.getId()); } else if (fieldName.equals("rules")) { List expectedRule = (List) expected; - List actualRule = resolveRules(JsonUtils.readObjects(actual.toString(), Object.class)); + List actualRule = JsonUtils.readObjects(actual.toString(), Rule.class); assertEquals(expectedRule, actualRule); } else if (fieldName.startsWith("rules") && (fieldName.endsWith("effect"))) { Effect expectedEffect = (Effect) expected; @@ -149,12 +150,6 @@ public class PolicyResourceTest extends EntityResourceTest } } - @Test - void post_PolicyWithoutPolicyType_400_badRequest(TestInfo test) { - CreatePolicy create = createRequest(test).withPolicyType(null); - assertResponse(() -> createEntity(create, ADMIN_AUTH_HEADERS), BAD_REQUEST, "[policyType must not be null]"); - } - @Test void post_validPolicies_as_admin_200_OK(TestInfo test) throws IOException { // Create valid policy @@ -402,12 +397,7 @@ public class PolicyResourceTest extends EntityResourceTest } private CreatePolicy createAccessControlPolicyWithRules(String name, List rules) { - return new CreatePolicy() - .withName(name) - .withDescription("description") - .withPolicyType(PolicyType.AccessControl) - .withRules(rules.stream().map(rule -> (Object) rule).collect(Collectors.toList())) - .withOwner(USER1_REF); + return new CreatePolicy().withName(name).withDescription("description").withRules(rules).withOwner(USER1_REF); } private void validateCondition(String expression) throws HttpResponseException { diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java index cf250e232db..2541bba8447 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java @@ -80,7 +80,6 @@ import org.openmetadata.catalog.type.ChangeDescription; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.ImageList; import org.openmetadata.catalog.type.MetadataOperation; -import org.openmetadata.catalog.type.PolicyType; import org.openmetadata.catalog.type.Profile; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.JsonUtils; @@ -748,10 +747,7 @@ public class TeamResourceTest extends EntityResourceTest { // Create a policy with the rule PolicyResourceTest policyResourceTest = new PolicyResourceTest(); CreatePolicy createPolicy = - policyResourceTest - .createRequest("TeamManagerPolicy", "", "", null) - .withPolicyType(PolicyType.AccessControl) - .withRules(List.of(rule)); + policyResourceTest.createRequest("TeamManagerPolicy", "", "", null).withRules(List.of(rule)); Policy policy = policyResourceTest.createEntity(createPolicy, ADMIN_AUTH_HEADERS); // Create TeamManager role with the policy to update team diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/security/policyevaluator/SubjectContextTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/security/policyevaluator/SubjectContextTest.java index e3539a88d30..13fe3df2c06 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/security/policyevaluator/SubjectContextTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/security/policyevaluator/SubjectContextTest.java @@ -216,8 +216,8 @@ public class SubjectContextTest { return policies; } - private static List getRules(String prefix, int count) { - List rules = new ArrayList<>(count); + private static List getRules(String prefix, int count) { + List rules = new ArrayList<>(count); for (int i = 1; i <= count; i++) { rules.add(new Rule().withName(prefix + "rule" + count)); } diff --git a/ingestion-core/src/metadata/_version.py b/ingestion-core/src/metadata/_version.py index ac8f1868396..03f27b82314 100644 --- a/ingestion-core/src/metadata/_version.py +++ b/ingestion-core/src/metadata/_version.py @@ -7,5 +7,5 @@ Provides metadata version information. from incremental import Version -__version__ = Version("metadata", 0, 12, 0, dev=17) +__version__ = Version("metadata", 0, 12, 0, dev=18) __all__ = ["__version__"] diff --git a/ingestion/src/metadata/ingestion/models/ometa_policy.py b/ingestion/src/metadata/ingestion/models/ometa_policy.py deleted file mode 100644 index cd0cee9b262..00000000000 --- a/ingestion/src/metadata/ingestion/models/ometa_policy.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2021 Collate -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Optional - -from pydantic import BaseModel - -from metadata.generated.schema.entity.data.location import Location -from metadata.generated.schema.entity.policies.policy import Policy - - -class OMetaPolicy(BaseModel): - policy: Policy - # A Lifecycle Policy may be associated with a specific Location such as S3 bucket. - location: Optional[Location] diff --git a/ingestion/src/metadata/ingestion/sink/metadata_rest.py b/ingestion/src/metadata/ingestion/sink/metadata_rest.py index c3c2e465a25..86467a0f9f9 100644 --- a/ingestion/src/metadata/ingestion/sink/metadata_rest.py +++ b/ingestion/src/metadata/ingestion/sink/metadata_rest.py @@ -46,7 +46,6 @@ from metadata.ingestion.lineage.sql_lineage import ( _create_lineage_by_table_name, get_lineage_by_query, ) -from metadata.ingestion.models.ometa_policy import OMetaPolicy from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable from metadata.ingestion.models.ometa_tag_category import OMetaTagAndCategory from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus @@ -121,8 +120,6 @@ class MetadataRestSink(Sink[Entity]): def write_record(self, record: Entity) -> None: if isinstance(record, OMetaDatabaseAndTable): self.write_tables(record) - elif isinstance(record, OMetaPolicy): - self.write_policies(record) elif isinstance(record, AddLineageRequest): self.write_lineage(record) elif isinstance(record, OMetaUserProfile): @@ -370,35 +367,6 @@ class MetadataRestSink(Sink[Entity]): f"Unexpected error writing db schema and table [{db_schema_and_table}]: {exc}" ) - def write_policies(self, ometa_policy: OMetaPolicy) -> None: - try: - created_location = None - if ometa_policy.location is not None: - created_location = self._create_location(ometa_policy.location) - logger.info(f"Successfully ingested Location {created_location.name}") - self.status.records_written(f"Location: {created_location.name}") - - policy_request = CreatePolicyRequest( - name=ometa_policy.policy.name, - displayName=ometa_policy.policy.displayName, - description=ometa_policy.policy.description, - owner=ometa_policy.policy.owner, - policyUrl=ometa_policy.policy.policyUrl, - policyType=ometa_policy.policy.policyType, - rules=ometa_policy.policy.rules, - location=created_location.id if created_location else None, - ) - created_policy = self.metadata.create_or_update(policy_request) - logger.info(f"Successfully ingested Policy {created_policy.name}") - self.status.records_written(f"Policy: {created_policy.name}") - - except (APIError, ValidationError) as err: - logger.debug(traceback.format_exc()) - logger.warning( - f"Failed to ingest Policy [{ometa_policy.policy.name}]: {err}" - ) - self.status.failure(f"Policy: {ometa_policy.policy.name}") - def _create_location(self, location: Location) -> Location: try: location_request = CreateLocationRequest( diff --git a/openmetadata-docs/content/main-concepts/metadata-standard/schemas/api/policies/createPolicy.md b/openmetadata-docs/content/main-concepts/metadata-standard/schemas/api/policies/createPolicy.md index b73df7b23a6..cbb4e85ac1f 100644 --- a/openmetadata-docs/content/main-concepts/metadata-standard/schemas/api/policies/createPolicy.md +++ b/openmetadata-docs/content/main-concepts/metadata-standard/schemas/api/policies/createPolicy.md @@ -13,7 +13,6 @@ slug: /main-concepts/metadata-standard/schemas/api/policies/createpolicy - **`displayName`** *(string)*: Title for this Policy. - **`description`**: A short description of the Policy, comprehensible to regular users. Refer to *../../type/basic.json#/definitions/markdown*. - **`owner`**: Owner of this Policy. Refer to *../../type/entityReference.json*. -- **`policyType`**: Refer to *../../entity/policies/policy.json#/definitions/policyType*. - **`rules`**: Refer to *../../entity/policies/policy.json#/definitions/rules*. - **`enabled`** *(boolean)*: Is the policy enabled. Default: `True`. - **`location`**: UUID of Location where this policy is applied. Refer to *../../type/basic.json#/definitions/uuid*. Default: `None`. diff --git a/openmetadata-docs/content/main-concepts/metadata-standard/schemas/entity/policies/policy.md b/openmetadata-docs/content/main-concepts/metadata-standard/schemas/entity/policies/policy.md index c9d72b787e4..4dc6e958414 100644 --- a/openmetadata-docs/content/main-concepts/metadata-standard/schemas/entity/policies/policy.md +++ b/openmetadata-docs/content/main-concepts/metadata-standard/schemas/entity/policies/policy.md @@ -16,7 +16,6 @@ slug: /main-concepts/metadata-standard/schemas/entity/policies/policy - **`description`**: A short description of the Policy, comprehensible to regular users. Refer to *../../type/basic.json#/definitions/markdown*. - **`owner`**: Owner of this Policy. Refer to *../../type/entityReference.json*. Default: `None`. - **`href`**: Link to the resource corresponding to this entity. Refer to *../../type/basic.json#/definitions/href*. -- **`policyType`**: Refer to *#/definitions/policyType*. - **`enabled`** *(boolean)*: Is the policy enabled. Default: `True`. - **`version`**: Metadata version of the Policy. Refer to *../../type/entityHistory.json#/definitions/entityVersion*. - **`updatedAt`**: Last update time corresponding to the new version of the Policy in Unix epoch time milliseconds. Refer to *../../type/basic.json#/definitions/timestamp*. @@ -27,7 +26,6 @@ slug: /main-concepts/metadata-standard/schemas/entity/policies/policy - **`deleted`** *(boolean)*: When `true` indicates the entity has been soft deleted. Default: `False`. ## Definitions -- **`policyType`** *(string)*: This schema defines the type used for describing different types of policies. Must be one of: `['AccessControl', 'Lifecycle']`. - **`rules`** *(array)*: A set of rules associated with the Policy. - **Items** diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/PoliciesPage/AddPolicyPage/AddPolicyPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/PoliciesPage/AddPolicyPage/AddPolicyPage.tsx index 2f5bc9c7dcb..aeb4190155b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/PoliciesPage/AddPolicyPage/AddPolicyPage.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/PoliciesPage/AddPolicyPage/AddPolicyPage.tsx @@ -33,7 +33,6 @@ import { ADD_POLICY_TEXT } from '../../../constants/HelperTextUtil'; import { CreatePolicy, Effect, - PolicyType, Rule, } from '../../../generated/api/policies/createPolicy'; import { @@ -84,7 +83,6 @@ const AddPolicyPage = () => { const data: CreatePolicy = { name, description, - policyType: PolicyType.AccessControl, rules: [condition ? { ...rest, condition } : rest], }; diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/PoliciesPage/policies.mock.ts b/openmetadata-ui/src/main/resources/ui/src/pages/PoliciesPage/policies.mock.ts index 3f1bf078028..b429ed43651 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/PoliciesPage/policies.mock.ts +++ b/openmetadata-ui/src/main/resources/ui/src/pages/PoliciesPage/policies.mock.ts @@ -8,7 +8,6 @@ export const POLICY_DATA = { description: 'Policy for Data Consumer to perform operations on metadata entities', href: 'http://localhost:8585/api/v1/policies/4b762714-8228-4a65-977c-86330c53ff5e', - policyType: 'AccessControl', enabled: true, version: 0.2, updatedAt: 1661494134803, diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/RolesPage/Roles.mock.ts b/openmetadata-ui/src/main/resources/ui/src/pages/RolesPage/Roles.mock.ts index e54d79eb9fa..61ec75ee632 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/RolesPage/Roles.mock.ts +++ b/openmetadata-ui/src/main/resources/ui/src/pages/RolesPage/Roles.mock.ts @@ -187,7 +187,6 @@ export const POLICY_LIST_WITH_PAGING = { description: 'Policy for Data Consumer to perform operations on metadata entities', href: 'http://localhost:8585/api/v1/policies/1ff8f95a-0fd7-4429-ba56-ea95ee582459', - policyType: 'AccessControl', enabled: true, version: 0.1, updatedAt: 1661318304992, @@ -212,7 +211,6 @@ export const POLICY_LIST_WITH_PAGING = { description: 'Policy for Data Steward Role to perform operations on metadata entities', href: 'http://localhost:8585/api/v1/policies/b0327d82-521f-4381-9f17-98c11408446f', - policyType: 'AccessControl', enabled: true, version: 0.1, updatedAt: 1661318304972, @@ -241,7 +239,6 @@ export const POLICY_LIST_WITH_PAGING = { displayName: 'Organization Policy', description: 'Policy for all the users of an organization.', href: 'http://localhost:8585/api/v1/policies/2a34e7ab-0edd-428f-8d91-e70033c3c204', - policyType: 'AccessControl', enabled: true, version: 0.1, updatedAt: 1661318304689, @@ -275,7 +272,6 @@ export const POLICY_LIST_WITH_PAGING = { description: 'Policy when attached to a team allows only users with in the team hierarchy to access the resources.', href: 'http://localhost:8585/api/v1/policies/9216e93f-72c4-4158-a75f-406d1c65d78f', - policyType: 'AccessControl', enabled: true, version: 0.8, updatedAt: 1661439183482,