diff --git a/Makefile b/Makefile index b65a70fe605..ebc9efe149a 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,8 @@ py_format_check: ## Check if Python sources are correctly formatted generate: ## Generate the pydantic models from the JSON Schemas to the ingestion module @echo "Running Datamodel Code Generator" @echo "Make sure to first run the install_dev recipe" - datamodel-codegen --input catalog-rest-service/src/main/resources/json --input-file-type jsonschema --output ingestion/src/metadata/generated --set-default-enum-member + mkdir -p ingestion/src/metadata/generated + datamodel-codegen --input catalog-rest-service/src/main/resources/json/schema --input-file-type jsonschema --output ingestion/src/metadata/generated/schema --set-default-enum-member $(MAKE) py_antlr $(MAKE) install @@ -162,7 +163,7 @@ core_generate: ## Generate the pydantic models from the JSON Schemas to the ing $(MAKE) core_install_dev mkdir -p ingestion-core/src/metadata/generated; \ . ingestion-core/venv/bin/activate; \ - datamodel-codegen --input catalog-rest-service/src/main/resources/json --input-file-type jsonschema --output ingestion-core/src/metadata/generated + datamodel-codegen --input catalog-rest-service/src/main/resources/json/schema --input-file-type jsonschema --output ingestion-core/src/metadata/generated/schema $(MAKE) core_py_antlr .PHONY: core_bump_version_dev diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java index 28a29576f39..20b933162e9 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/Entity.java @@ -51,7 +51,7 @@ public final class Entity { // Canonical entity name to corresponding EntityRepository map private static final Map> ENTITY_REPOSITORY_MAP = new HashMap<>(); - // List of entities + // List of all the entities private static final List ENTITY_LIST = new ArrayList<>(); // Common field names @@ -152,7 +152,7 @@ public final class Entity { entityRepository.getClass().getSimpleName()); } - public static List listEntities() { + public static List getEntityList() { return Collections.unmodifiableList(ENTITY_LIST); } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/ResourceRegistry.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/ResourceRegistry.java new file mode 100644 index 00000000000..afc44665547 --- /dev/null +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/ResourceRegistry.java @@ -0,0 +1,23 @@ +package org.openmetadata.catalog; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import org.openmetadata.catalog.type.ResourceDescriptor; + +public class ResourceRegistry { + private static final ResourceRegistry registry = new ResourceRegistry(); + private static final List RESOURCE_DESCRIPTORS = new ArrayList<>(); + + private ResourceRegistry() {} + + public static void add(List resourceDescriptors) { + RESOURCE_DESCRIPTORS.addAll(resourceDescriptors); + RESOURCE_DESCRIPTORS.sort(Comparator.comparing(ResourceDescriptor::getName)); + } + + public static List listResourceDescriptors() { + return Collections.unmodifiableList(RESOURCE_DESCRIPTORS); + } +} diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java index 764cda38c95..ef2d4c60db0 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/databases/TableResource.java @@ -13,6 +13,8 @@ package org.openmetadata.catalog.resources.databases; +import static org.openmetadata.catalog.type.MetadataOperation.*; + import io.swagger.annotations.Api; import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/policies/PolicyResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/policies/PolicyResource.java index 08eabf630b7..a80f25cf367 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/policies/PolicyResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/policies/PolicyResource.java @@ -23,8 +23,6 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import javax.json.JsonPatch; import javax.validation.Valid; @@ -47,8 +45,10 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; +import org.apache.maven.shared.utils.io.IOUtil; import org.openmetadata.catalog.CatalogApplicationConfig; import org.openmetadata.catalog.Entity; +import org.openmetadata.catalog.ResourceRegistry; import org.openmetadata.catalog.api.policies.CreatePolicy; import org.openmetadata.catalog.entity.policies.Policy; import org.openmetadata.catalog.jdbi3.CollectionDAO; @@ -61,6 +61,9 @@ import org.openmetadata.catalog.security.policyevaluator.PolicyEvaluator; import org.openmetadata.catalog.type.EntityHistory; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.Include; +import org.openmetadata.catalog.type.ResourceDescriptor; +import org.openmetadata.catalog.util.EntityUtil; +import org.openmetadata.catalog.util.JsonUtils; import org.openmetadata.catalog.util.ResultList; @Slf4j @@ -71,7 +74,6 @@ import org.openmetadata.catalog.util.ResultList; @Collection(name = "policies") public class PolicyResource extends EntityResource { public static final String COLLECTION_PATH = "v1/policies/"; - public static final List RESOURCES = new ArrayList<>(); @Override public Policy addHref(UriInfo uriInfo, Policy policy) { @@ -92,6 +94,24 @@ public class PolicyResource extends EntityResource { // Load any existing rules from database, before loading seed data. policyEvaluator.load(); dao.initSeedDataFromResources(); + initResourceDescriptors(); + } + + /** Initialize list of resources and the corresponding operations */ + private void initResourceDescriptors() throws IOException { + List jsonDataFiles = EntityUtil.getJsonDataResources(".*json/data/ResourceDescriptors.json$"); + if (jsonDataFiles.size() != 1) { + LOG.warn("Invalid number of jsonDataFiles {}. Only one expected.", jsonDataFiles.size()); + return; + } + String jsonDataFile = jsonDataFiles.get(0); + try { + String json = IOUtil.toString(getClass().getClassLoader().getResourceAsStream(jsonDataFile)); + List resourceDetails = JsonUtils.readObjects(json, ResourceDescriptor.class); + ResourceRegistry.add(resourceDetails); + } catch (Exception e) { + LOG.warn("Failed to initialize the {} from file {}", entityType, jsonDataFile, e); + } } public static class PolicyList extends ResultList { @@ -105,6 +125,17 @@ public class PolicyResource extends EntityResource { } } + public static class ResourceDescriptorList extends ResultList { + @SuppressWarnings("unused") + ResourceDescriptorList() { + // Empty constructor needed for deserialization + } + + public ResourceDescriptorList(List data) { + super(data, null, null, data.size()); + } + } + public static final String FIELDS = "owner,location"; @GET @@ -283,16 +314,9 @@ public class PolicyResource extends EntityResource { responseCode = "404", description = "Policy for instance {id} and version {version} is" + " " + "not found") }) - public List listPolicyResources(@Context UriInfo uriInfo, @Context SecurityContext securityContext) - throws IOException { - if (RESOURCES.isEmpty()) { - // Load set of resource types - RESOURCES.addAll(Entity.listEntities()); - RESOURCES.add("lineage"); - RESOURCES.add("feed"); - Collections.sort(RESOURCES); - } - return RESOURCES; + public ResultList listPolicyResources( + @Context UriInfo uriInfo, @Context SecurityContext securityContext) throws IOException { + return new ResourceDescriptorList(ResourceRegistry.listResourceDescriptors()); } @POST diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/security/policyevaluator/PolicyEvaluator.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/security/policyevaluator/PolicyEvaluator.java index 04b2e5a9c9c..0a868e6380e 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/security/policyevaluator/PolicyEvaluator.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/security/policyevaluator/PolicyEvaluator.java @@ -15,6 +15,7 @@ package org.openmetadata.catalog.security.policyevaluator; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import lombok.NonNull; @@ -55,7 +56,7 @@ import org.openmetadata.catalog.util.JsonUtils; public class PolicyEvaluator { private PolicyRepository policyRepository; - private final ConcurrentHashMap policyToRules = new ConcurrentHashMap<>(); + private final Map policyToRules = new ConcurrentHashMap<>(); private final RulesEngine checkPermissionRulesEngine; private final RulesEngine allowedOperationsRulesEngine; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonPatchUtils.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonPatchUtils.java index c488a90b8f5..ea2a13f15ca 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonPatchUtils.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/JsonPatchUtils.java @@ -34,7 +34,7 @@ public class JsonPatchUtils { return MetadataOperation.EDIT_OWNER; } if (path.startsWith("/users")) { // Ability to update users within a team. - return MetadataOperation.TEAM_EDIT_USERS; + return MetadataOperation.EDIT_USERS; } return null; } diff --git a/catalog-rest-service/src/main/resources/json/data/ResourceDescriptors.json b/catalog-rest-service/src/main/resources/json/data/ResourceDescriptors.json new file mode 100644 index 00000000000..fb8ba637f51 --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/data/ResourceDescriptors.json @@ -0,0 +1,361 @@ +[ + { + "name" : "bot", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name": "chart", + "operations": [ + "Create", + "Delete", + "ViewAll", + "ViewUsage", + "EditAll", + "EditDescription", + "EditTags", + "EditOwner", + "EditTier", + "EditCustomFields", + "EditLineage" + ] + }, + { + "name" : "dashboard", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "ViewUsage", + "EditAll", + "EditDescription", + "EditTags", + "EditOwner", + "EditTier", + "EditCustomFields", + "EditLineage" + ] + }, + { + "name" : "dashboardService", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "database", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "databaseSchema", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "databaseService", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "glossary", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "glossaryTerm", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "ingestionPipeline", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "location", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "messagingService", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "metrics", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "mlmodel", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "ViewUsage", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "mlmodelService", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "pipeline", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields", + "EditLineage" + ] + }, + { + "name" : "pipelineService", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "policy", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "report", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "role", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "storageService", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "table", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "ViewUsage", + "ViewTests", + "ViewQueries", + "ViewDataProfile", + "ViewSampleData", + "EditAll", + "EditDescription", + "EditTags", + "EditOwner", + "EditTier", + "EditCustomFields", + "EditTests", + "EditQueries", + "EditDataProfile", + "EditSampleData", + "EditLineage" + ] + }, + { + "name" : "tag", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "tagCategory", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "team", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields", + "EditUsers" + ] + }, + { + "name" : "topic", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "type", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "user", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "webhook", + "operations" : [ + "Create", + "Delete", + "ViewAll", + "EditAll", + "EditDescription", + "EditCustomFields" + ] + }, + { + "name" : "events", + "operations" : [ + "ViewAll" + ] + }, + { + "name" : "feed", + "operations" : [ + "Create", + "Delete", + "ViewAll" + ] + } +] \ No newline at end of file diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/policies/accessControl/resourceDescriptor.json b/catalog-rest-service/src/main/resources/json/schema/entity/policies/accessControl/resourceDescriptor.json new file mode 100644 index 00000000000..cf4bf519d9b --- /dev/null +++ b/catalog-rest-service/src/main/resources/json/schema/entity/policies/accessControl/resourceDescriptor.json @@ -0,0 +1,51 @@ +{ + "$id": "https://open-metadata.org/schema/entity/data/policies/accessControl/resourceDescriptor.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ResourceDescriptor", + "description": "Resource descriptor", + "type": "object", + "javaType": "org.openmetadata.catalog.type.ResourceDescriptor", + "definitions": { + "operation": { + "javaType": "org.openmetadata.catalog.type.MetadataOperation", + "description": "This schema defines all possible operations on metadata of entities in OpenMetadata.", + "type": "string", + "enum": [ + "Create", + "Delete", + "ViewAll", + "ViewUsage", + "ViewTests", + "ViewQueries", + "ViewDataProfile", + "ViewSampleData", + "EditAll", + "EditDescription", + "EditTags", + "EditOwner", + "EditTier", + "EditCustomFields", + "EditLineage", + "EditReviewers", + "EditTests", + "EditQueries", + "EditDataProfile", + "EditSampleData", + "EditUsers" + ] + } + }, + "properties": { + "name": { + "description": "Name of the resource. For entity related resources, resource name is same as the entity name. Some resources such as lineage are not entities but are resources.", + "type": "string" + }, + "operations": { + "description": "List of operations supported by the resource.", + "type": "array", + "items": { + "$ref": "#/definitions/operation" + } + } + } +} diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/policies/accessControl/rule.json b/catalog-rest-service/src/main/resources/json/schema/entity/policies/accessControl/rule.json index dd9c5c06eef..9467126cc26 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/policies/accessControl/rule.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/policies/accessControl/rule.json @@ -5,36 +5,6 @@ "description": "Describes an Access Control Rule for OpenMetadata Metadata Operations. All non-null user (subject) and entity (object) attributes are evaluated with logical AND.", "type": "object", "javaType": "org.openmetadata.catalog.entity.policies.accessControl.Rule", - "definitions": { - "operation": { - "javaType": "org.openmetadata.catalog.type.MetadataOperation", - "description": "This schema defines all possible operations on metadata of data entities.", - "type": "string", - "enum": [ - "Create", - "Delete", - "ViewAll", - "ViewUsage", - "ViewTests", - "TableViewQueries", - "TableViewDataProfile", - "TableViewSampleData", - "EditAll", - "EditDescription", - "EditTags", - "EditOwner", - "EditTier", - "EditCustomFields", - "EditLineage", - "EditReviewers", - "EditTests", - "TableEditQueries", - "TableEditDataProfile", - "TableEditSampleData", - "TeamEditUsers" - ] - } - }, "properties": { "name": { "description": "Name for this Rule.", @@ -56,7 +26,7 @@ }, "operation": { "description": "Operation on the entity.", - "$ref": "#/definitions/operation", + "$ref": "resourceDescriptor.json#/definitions/operation", "default": null }, "allow": { diff --git a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json index 8fc76f0f731..0fb0fc0fc01 100644 --- a/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json +++ b/catalog-rest-service/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json @@ -65,7 +65,7 @@ }, "queryTag": { "title": "Query Tag", - "description": "Session query tag used to monitor usage on snoflake. To use a query tag snowflake user should have enough privileges to alter the session.", + "description": "Session query tag used to monitor usage on snowflake. To use a query tag snowflake user should have enough privileges to alter the session.", "type": "string" }, "privateKey": { diff --git a/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnToMatchSet.json b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnToMatchSet.json index 07f4d613d19..99d43aee12f 100644 --- a/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnToMatchSet.json +++ b/catalog-rest-service/src/main/resources/json/schema/tests/table/tableColumnToMatchSet.json @@ -11,7 +11,7 @@ "type": "string" }, "ordered": { - "description": "Wether or not to considered the order of the list when performing the match", + "description": "Whether or not to considered the order of the list when performing the match", "type": "boolean", "default": false } diff --git a/catalog-rest-service/src/main/resources/json/schema/type/usageRequest.json b/catalog-rest-service/src/main/resources/json/schema/type/usageRequest.json index e01cef8ce89..f68434d117c 100644 --- a/catalog-rest-service/src/main/resources/json/schema/type/usageRequest.json +++ b/catalog-rest-service/src/main/resources/json/schema/type/usageRequest.json @@ -2,7 +2,7 @@ "$id": "https://open-metadata.org/schema/entity/data/usageRequest.json", "$schema": "http://json-schema.org/draft-07/schema#", "title": "Usage Request", - "description": "This schema defines type of table usage request used to publish the usage count on a perticular date", + "description": "This schema defines type of table usage request used to publish the usage count on a particular date", "javaType": "org.openmetadata.catalog.type.UsageRequest", "type": "object", "properties": { diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/permissions/PermissionsResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/permissions/PermissionsResourceTest.java index cc428b7973f..6a80ed9240b 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/permissions/PermissionsResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/permissions/PermissionsResourceTest.java @@ -94,14 +94,14 @@ class PermissionsResourceTest extends CatalogApplicationTest { dataStewardPermissions.put(MetadataOperation.EDIT_OWNER, Boolean.TRUE); dataStewardPermissions.put(MetadataOperation.EDIT_TAGS, Boolean.TRUE); // put(MetadataOperation.DecryptTokens, Boolean.FALSE); - dataStewardPermissions.put(MetadataOperation.TEAM_EDIT_USERS, Boolean.FALSE); + dataStewardPermissions.put(MetadataOperation.EDIT_USERS, Boolean.FALSE); dataConsumerPermissions.put(MetadataOperation.EDIT_DESCRIPTION, Boolean.TRUE); dataConsumerPermissions.put(MetadataOperation.EDIT_LINEAGE, Boolean.FALSE); dataConsumerPermissions.put(MetadataOperation.EDIT_OWNER, Boolean.TRUE); dataConsumerPermissions.put(MetadataOperation.EDIT_TAGS, Boolean.TRUE); // put(MetadataOperation.DecryptTokens, Boolean.FALSE); - dataConsumerPermissions.put(MetadataOperation.TEAM_EDIT_USERS, Boolean.FALSE); + dataConsumerPermissions.put(MetadataOperation.EDIT_USERS, Boolean.FALSE); return Stream.of( Arguments.of(TestUtils.ADMIN_USER_NAME, adminPermissions), diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java index 2cfe9a345e6..bb9822b6049 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/policies/PolicyResourceTest.java @@ -15,7 +15,7 @@ package org.openmetadata.catalog.resources.policies; import static javax.ws.rs.core.Response.Status.BAD_REQUEST; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS; import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE; import static org.openmetadata.catalog.util.TestUtils.assertListNotNull; @@ -47,11 +47,13 @@ import org.openmetadata.catalog.exception.CatalogExceptionMessage; import org.openmetadata.catalog.resources.EntityResourceTest; import org.openmetadata.catalog.resources.locations.LocationResourceTest; import org.openmetadata.catalog.resources.policies.PolicyResource.PolicyList; +import org.openmetadata.catalog.resources.policies.PolicyResource.ResourceDescriptorList; import org.openmetadata.catalog.type.ChangeDescription; import org.openmetadata.catalog.type.EntityReference; import org.openmetadata.catalog.type.FieldChange; import org.openmetadata.catalog.type.MetadataOperation; import org.openmetadata.catalog.type.PolicyType; +import org.openmetadata.catalog.type.ResourceDescriptor; import org.openmetadata.catalog.util.EntityUtil; import org.openmetadata.catalog.util.JsonUtils; import org.openmetadata.catalog.util.PolicyUtils; @@ -189,10 +191,17 @@ public class PolicyResourceTest extends EntityResourceTest @Test void get_policyResources() throws HttpResponseException { // Get list of policy resources and make sure it has all the entities and other resources - List resources = getPolicyResources(ADMIN_AUTH_HEADERS); - List entities = Entity.listEntities(); - assertTrue(resources.containsAll(entities)); - assertTrue(resources.contains("lineage")); + ResourceDescriptorList actualResourceDescriptors = getPolicyResources(ADMIN_AUTH_HEADERS); + assertNotNull(actualResourceDescriptors.getData()); + System.out.println(actualResourceDescriptors.getData()); + + // Ensure all entities are captured in resource descriptor list + List entities = Entity.getEntityList(); + for (String entity : entities) { + ResourceDescriptor resourceDescriptor = + actualResourceDescriptors.getData().stream().filter(rd -> rd.getName().equals(entity)).findFirst().get(); + assertNotNull(resourceDescriptor); + } } @Override @@ -231,8 +240,8 @@ public class PolicyResourceTest extends EntityResourceTest return TestUtils.post(getResource("locations"), createLocation, Location.class, ADMIN_AUTH_HEADERS); } - public final List getPolicyResources(Map authHeaders) throws HttpResponseException { + public final ResourceDescriptorList getPolicyResources(Map authHeaders) throws HttpResponseException { WebTarget target = getResource(collectionName + "/resources"); - return (List) TestUtils.get(target, List.class, authHeaders); + return TestUtils.get(target, ResourceDescriptorList.class, authHeaders); } } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java index 883d860aa54..c19a96756ad 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/teams/TeamResourceTest.java @@ -215,7 +215,7 @@ public class TeamResourceTest extends EntityResourceTest { patchEntity( team.getId(), originalJson, team, SecurityUtil.authHeaders(randomUserName + "@open-metadata.org")), FORBIDDEN, - CatalogExceptionMessage.noPermission(randomUserName, "TeamEditUsers")); + CatalogExceptionMessage.noPermission(randomUserName, "EditUsers")); // Ensure user with UpdateTeam permission can add users to a team. User teamManagerUser = createTeamManager(test); @@ -459,10 +459,7 @@ public class TeamResourceTest extends EntityResourceTest { private User createTeamManager(TestInfo testInfo) throws HttpResponseException { // Create a rule that can update team Rule rule = - new Rule() - .withName("TeamManagerPolicy-UpdateTeam") - .withAllow(true) - .withOperation(MetadataOperation.TEAM_EDIT_USERS); + new Rule().withName("TeamManagerPolicy-UpdateTeam").withAllow(true).withOperation(MetadataOperation.EDIT_USERS); // Create a policy with the rule PolicyResourceTest policyResourceTest = new PolicyResourceTest(); diff --git a/ingestion-core/src/metadata/_version.py b/ingestion-core/src/metadata/_version.py index 1218a920918..dd1a8293b19 100644 --- a/ingestion-core/src/metadata/_version.py +++ b/ingestion-core/src/metadata/_version.py @@ -7,5 +7,5 @@ Provides metadata version information. from incremental import Version -__version__ = Version("metadata", 0, 12, 0, dev=2) +__version__ = Version("metadata", 0, 12, 0, dev=3) __all__ = ["__version__"] diff --git a/ingestion/src/metadata/cmd.py b/ingestion/src/metadata/cmd.py index be52c1be411..39c544dee71 100644 --- a/ingestion/src/metadata/cmd.py +++ b/ingestion/src/metadata/cmd.py @@ -24,7 +24,6 @@ from metadata.cli.backup import run_backup from metadata.cli.docker import run_docker from metadata.cli.ingest import run_ingest from metadata.config.common import load_config_file -from metadata.ingestion.api.workflow import Workflow from metadata.orm_profiler.api.workflow import ProfilerWorkflow from metadata.utils.logger import cli_logger, set_loggers_level diff --git a/ingestion/src/metadata/ingestion/api/workflow.py b/ingestion/src/metadata/ingestion/api/workflow.py index 6bd5f36d2de..eec75be755b 100644 --- a/ingestion/src/metadata/ingestion/api/workflow.py +++ b/ingestion/src/metadata/ingestion/api/workflow.py @@ -19,14 +19,6 @@ from metadata.config.workflow import get_source_dir from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) -from metadata.generated.schema.entity.services.dashboardService import ( - DashboardConnection, -) -from metadata.generated.schema.entity.services.databaseService import DatabaseConnection -from metadata.generated.schema.entity.services.messagingService import ( - MessagingConnection, -) -from metadata.generated.schema.entity.services.metadataService import MetadataConnection from metadata.generated.schema.metadataIngestion.workflow import ( OpenMetadataWorkflowConfig, ) diff --git a/ingestion/src/metadata/ingestion/models/table_metadata.py b/ingestion/src/metadata/ingestion/models/table_metadata.py index 2adcdbd8f38..72225a60c11 100644 --- a/ingestion/src/metadata/ingestion/models/table_metadata.py +++ b/ingestion/src/metadata/ingestion/models/table_metadata.py @@ -14,7 +14,6 @@ from typing import List, Optional from pydantic import BaseModel -from metadata.generated.schema.entity.data.chart import Chart from metadata.generated.schema.entity.data.mlmodel import ( MlFeature, MlHyperParameter, diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py index ccfb3376d22..37ab226fd0b 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/lineage_mixin.py @@ -19,7 +19,6 @@ from typing import Any, Dict, Generic, Optional, Type, TypeVar, Union from pydantic import BaseModel from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest -from metadata.generated.schema.entity.data.table import Table from metadata.ingestion.ometa.client import REST, APIError from metadata.ingestion.ometa.utils import get_entity_type, ometa_logger diff --git a/ingestion/src/metadata/ingestion/sink/elasticsearch.py b/ingestion/src/metadata/ingestion/sink/elasticsearch.py index 380045d8412..e546e773146 100644 --- a/ingestion/src/metadata/ingestion/sink/elasticsearch.py +++ b/ingestion/src/metadata/ingestion/sink/elasticsearch.py @@ -20,13 +20,12 @@ from elasticsearch import Elasticsearch from elasticsearch.connection import create_ssl_context from metadata.config.common import ConfigModel -from metadata.generated.schema.entity.data.chart import Chart from metadata.generated.schema.entity.data.dashboard import Dashboard from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm from metadata.generated.schema.entity.data.mlmodel import MlModel -from metadata.generated.schema.entity.data.pipeline import Pipeline, Task +from metadata.generated.schema.entity.data.pipeline import Pipeline from metadata.generated.schema.entity.data.table import Column, Table from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase.py index 1cb0c4b9e10..9d6021d36a0 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase.py @@ -41,17 +41,12 @@ from metadata.generated.schema.metadataIngestion.workflow import ( from metadata.generated.schema.type.entityLineage import EntitiesEdge from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.api.source import InvalidSourceException -from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource from metadata.ingestion.source.database.common_db_source import SQLSourceStatus from metadata.utils import fqn from metadata.utils.connections import get_connection from metadata.utils.filters import filter_by_chart -from metadata.utils.helpers import ( - get_chart_entities_from_id, - get_standard_chart_type, - replace_special_with, -) +from metadata.utils.helpers import get_standard_chart_type, replace_special_with from metadata.utils.logger import ingestion_logger from metadata.utils.sql_lineage import search_table_entities diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset.py b/ingestion/src/metadata/ingestion/source/dashboard/superset.py index b62aa4506c4..d969ab0e92b 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset.py @@ -42,7 +42,7 @@ from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.api.source import InvalidSourceException, SourceStatus from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource from metadata.utils import fqn -from metadata.utils.helpers import get_chart_entities_from_id, get_standard_chart_type +from metadata.utils.helpers import get_standard_chart_type from metadata.utils.logger import ingestion_logger logger = ingestion_logger() diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau.py index 1360d42248b..04a37682121 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau.py @@ -40,7 +40,6 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata OpenMetadataConnection, ) from metadata.generated.schema.entity.tags.tagCategory import Tag -from metadata.generated.schema.entity.teams.user import User from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) @@ -52,7 +51,7 @@ from metadata.ingestion.models.ometa_tag_category import OMetaTagAndCategory from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource from metadata.utils import fqn from metadata.utils.filters import filter_by_chart -from metadata.utils.helpers import get_chart_entities_from_id, get_standard_chart_type +from metadata.utils.helpers import get_standard_chart_type from metadata.utils.logger import ingestion_logger logger = ingestion_logger() diff --git a/ingestion/src/metadata/ingestion/source/database/common_db_source.py b/ingestion/src/metadata/ingestion/source/database/common_db_source.py index c4e672af955..23fc5a2b2a4 100644 --- a/ingestion/src/metadata/ingestion/source/database/common_db_source.py +++ b/ingestion/src/metadata/ingestion/source/database/common_db_source.py @@ -33,7 +33,6 @@ from metadata.generated.schema.entity.data.table import Table, TableType from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) -from metadata.generated.schema.entity.tags.tagCategory import Tag from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import ( DatabaseServiceMetadataPipeline, ) @@ -41,12 +40,6 @@ from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) from metadata.generated.schema.type.entityReference import EntityReference -from metadata.generated.schema.type.tagLabel import ( - LabelType, - State, - TagLabel, - TagSource, -) from metadata.ingestion.models.ometa_tag_category import OMetaTagAndCategory from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.database_service import ( @@ -57,7 +50,7 @@ from metadata.ingestion.source.database.sql_column_handler import SqlColumnHandl from metadata.ingestion.source.database.sqlalchemy_source import SqlAlchemySource from metadata.utils import fqn from metadata.utils.connections import get_connection, test_connection -from metadata.utils.filters import filter_by_database, filter_by_schema, filter_by_table +from metadata.utils.filters import filter_by_schema, filter_by_table from metadata.utils.logger import ingestion_logger from metadata.utils.sql_lineage import ( get_lineage_by_query, diff --git a/ingestion/src/metadata/ingestion/source/database/database_service.py b/ingestion/src/metadata/ingestion/source/database/database_service.py index 6ccbf79969c..c6ceeb2c85b 100644 --- a/ingestion/src/metadata/ingestion/source/database/database_service.py +++ b/ingestion/src/metadata/ingestion/source/database/database_service.py @@ -44,7 +44,7 @@ from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) from metadata.generated.schema.type import basic -from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName +from metadata.generated.schema.type.basic import FullyQualifiedEntityName from metadata.generated.schema.type.tagLabel import ( LabelType, State, diff --git a/ingestion/src/metadata/ingestion/source/database/datalake.py b/ingestion/src/metadata/ingestion/source/database/datalake.py index 1599400ee57..6ad5d3d6858 100644 --- a/ingestion/src/metadata/ingestion/source/database/datalake.py +++ b/ingestion/src/metadata/ingestion/source/database/datalake.py @@ -43,7 +43,6 @@ from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) from metadata.generated.schema.type.entityReference import EntityReference -from metadata.ingestion.api.common import Entity from metadata.ingestion.api.source import InvalidSourceException, SourceStatus from metadata.ingestion.models.ometa_tag_category import OMetaTagAndCategory from metadata.ingestion.ometa.ometa_api import OpenMetadata diff --git a/ingestion/src/metadata/ingestion/source/database/glue.py b/ingestion/src/metadata/ingestion/source/database/glue.py index c00369d4224..944859d70d9 100755 --- a/ingestion/src/metadata/ingestion/source/database/glue.py +++ b/ingestion/src/metadata/ingestion/source/database/glue.py @@ -16,7 +16,6 @@ from typing import Iterable, List, Optional from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema from metadata.generated.schema.entity.data.location import Location, LocationType -from metadata.generated.schema.entity.data.pipeline import Task from metadata.generated.schema.entity.data.table import Column, Table, TableType from metadata.generated.schema.entity.services.connections.database.glueConnection import ( GlueConnection, diff --git a/ingestion/src/metadata/ingestion/source/database/mssql.py b/ingestion/src/metadata/ingestion/source/database/mssql.py index 7987a627581..1c40ddb2526 100644 --- a/ingestion/src/metadata/ingestion/source/database/mssql.py +++ b/ingestion/src/metadata/ingestion/source/database/mssql.py @@ -25,7 +25,6 @@ from metadata.generated.schema.metadataIngestion.workflow import ( ) from metadata.ingestion.api.source import InvalidSourceException from metadata.ingestion.source.database.common_db_source import CommonDbSourceService -from metadata.utils.connections import get_connection from metadata.utils.filters import filter_by_database from metadata.utils.logger import ingestion_logger diff --git a/ingestion/src/metadata/ingestion/source/database/oracle.py b/ingestion/src/metadata/ingestion/source/database/oracle.py index af9fe477a0f..65977fb7d93 100644 --- a/ingestion/src/metadata/ingestion/source/database/oracle.py +++ b/ingestion/src/metadata/ingestion/source/database/oracle.py @@ -13,7 +13,6 @@ from sqlalchemy.engine.reflection import Inspector -from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.services.connections.database.oracleConnection import ( OracleConnection, ) @@ -23,7 +22,6 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) -from metadata.generated.schema.type.entityReference import EntityReference from metadata.ingestion.api.source import InvalidSourceException from metadata.ingestion.source.database.common_db_source import CommonDbSourceService diff --git a/ingestion/src/metadata/ingestion/source/database/postgres.py b/ingestion/src/metadata/ingestion/source/database/postgres.py index 7cec7d5ee10..725c14a239f 100644 --- a/ingestion/src/metadata/ingestion/source/database/postgres.py +++ b/ingestion/src/metadata/ingestion/source/database/postgres.py @@ -15,7 +15,6 @@ from typing import Iterable from sqlalchemy.engine.reflection import Inspector from sqlalchemy.inspection import inspect -from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.services.connections.database.postgresConnection import ( PostgresConnection, ) @@ -27,10 +26,8 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) -from metadata.generated.schema.type.entityReference import EntityReference -from metadata.ingestion.api.source import InvalidSourceException, SourceStatus +from metadata.ingestion.api.source import InvalidSourceException from metadata.ingestion.source.database.common_db_source import CommonDbSourceService -from metadata.utils.connections import get_connection from metadata.utils.filters import filter_by_database from metadata.utils.logger import ingestion_logger diff --git a/ingestion/src/metadata/ingestion/source/database/redshift_usage.py b/ingestion/src/metadata/ingestion/source/database/redshift_usage.py index dd08e0b256a..693edff4e9b 100644 --- a/ingestion/src/metadata/ingestion/source/database/redshift_usage.py +++ b/ingestion/src/metadata/ingestion/source/database/redshift_usage.py @@ -18,9 +18,6 @@ from typing import Iterator, Union from metadata.generated.schema.entity.services.connections.database.redshiftConnection import ( RedshiftConnection, ) -from metadata.generated.schema.entity.services.databaseService import ( - DatabaseServiceType, -) from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake_usage.py b/ingestion/src/metadata/ingestion/source/database/snowflake_usage.py index c60978be319..b2079f864b6 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake_usage.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake_usage.py @@ -26,7 +26,6 @@ from metadata.generated.schema.entity.services.databaseService import ( from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) -from metadata.generated.schema.metadataIngestion.workflow import WorkflowConfig from metadata.generated.schema.type.tableQuery import TableQuery from metadata.ingestion.api.source import InvalidSourceException diff --git a/ingestion/src/metadata/ingestion/source/database/sqlalchemy_source.py b/ingestion/src/metadata/ingestion/source/database/sqlalchemy_source.py index c999ca074eb..2dabe5289a0 100644 --- a/ingestion/src/metadata/ingestion/source/database/sqlalchemy_source.py +++ b/ingestion/src/metadata/ingestion/source/database/sqlalchemy_source.py @@ -17,14 +17,13 @@ from typing import List, Optional, Set, Tuple from sqlalchemy.engine import Engine from sqlalchemy.engine.reflection import Inspector -from metadata.generated.schema.entity.data.table import Column, DataModel, Table +from metadata.generated.schema.entity.data.table import Column, DataModel from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import ( DatabaseServiceMetadataPipeline, ) from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) -from metadata.generated.schema.type.tagLabel import TagLabel from metadata.ingestion.models.topology import TopologyContext from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.database_service import SQLSourceStatus diff --git a/ingestion/src/metadata/ingestion/source/messaging/messaging_service.py b/ingestion/src/metadata/ingestion/source/messaging/messaging_service.py index 8230f049ed3..a32eaa4d981 100644 --- a/ingestion/src/metadata/ingestion/source/messaging/messaging_service.py +++ b/ingestion/src/metadata/ingestion/source/messaging/messaging_service.py @@ -18,13 +18,8 @@ from typing import Any, Iterable, List, Optional from confluent_kafka.admin import AdminClient, ConfigResource -from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest from metadata.generated.schema.api.data.createTopic import CreateTopicRequest -from metadata.generated.schema.entity.data.topic import ( - SchemaType, - Topic, - TopicSampleData, -) +from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) @@ -38,7 +33,7 @@ from metadata.generated.schema.metadataIngestion.messagingServiceMetadataPipelin from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) -from metadata.ingestion.api.source import InvalidSourceException, Source, SourceStatus +from metadata.ingestion.api.source import Source, SourceStatus from metadata.ingestion.api.topology_runner import TopologyRunnerMixin from metadata.ingestion.models.topology import ( NodeStage, diff --git a/ingestion/src/metadata/ingestion/source/metadata/amundsen.py b/ingestion/src/metadata/ingestion/source/metadata/amundsen.py index 8cdf4aad50f..5a7da0de039 100644 --- a/ingestion/src/metadata/ingestion/source/metadata/amundsen.py +++ b/ingestion/src/metadata/ingestion/source/metadata/amundsen.py @@ -50,15 +50,10 @@ from metadata.ingestion.api.source import InvalidSourceException, Source, Source from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable from metadata.ingestion.models.ometa_tag_category import OMetaTagAndCategory from metadata.ingestion.models.user import OMetaUserProfile -from metadata.ingestion.ometa.client import APIError from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.utils import fqn from metadata.utils.column_type_parser import ColumnTypeParser -from metadata.utils.helpers import ( - get_chart_entities_from_id, - get_dashboard_service_or_create, - get_standard_chart_type, -) +from metadata.utils.helpers import get_chart_entities_from_id, get_standard_chart_type from metadata.utils.logger import ingestion_logger from metadata.utils.neo4j_helper import Neo4JConfig, Neo4jHelper from metadata.utils.sql_queries import ( diff --git a/ingestion/src/metadata/ingestion/source/metadata/atlas.py b/ingestion/src/metadata/ingestion/source/metadata/atlas.py index 8f29aa0f899..0055b8cd725 100644 --- a/ingestion/src/metadata/ingestion/source/metadata/atlas.py +++ b/ingestion/src/metadata/ingestion/source/metadata/atlas.py @@ -8,12 +8,8 @@ from typing import Any, Dict, Iterable, List import yaml from importlib_metadata import SelectableGroups -from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest from metadata.generated.schema.api.data.createTopic import CreateTopicRequest from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest -from metadata.generated.schema.api.services.createDatabaseService import ( - CreateDatabaseServiceRequest, -) from metadata.generated.schema.entity.data.database import Database from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema from metadata.generated.schema.entity.data.pipeline import Pipeline @@ -24,12 +20,8 @@ from metadata.generated.schema.entity.services.connections.metadata.atlasConnect from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, ) -from metadata.generated.schema.entity.services.databaseService import ( - DatabaseService, - DatabaseServiceType, -) +from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.entity.services.messagingService import MessagingService -from metadata.generated.schema.entity.services.metadataService import MetadataService from metadata.generated.schema.metadataIngestion.workflow import ( Source as WorkflowSource, ) diff --git a/ingestion/src/metadata/ingestion/source/metadata/migrate.py b/ingestion/src/metadata/ingestion/source/metadata/migrate.py index e025410cbe7..46e5c2020f3 100644 --- a/ingestion/src/metadata/ingestion/source/metadata/migrate.py +++ b/ingestion/src/metadata/ingestion/source/metadata/migrate.py @@ -17,7 +17,6 @@ from metadata.generated.schema.entity.data.dashboard import Dashboard from metadata.generated.schema.entity.data.glossary import Glossary from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm from metadata.generated.schema.entity.data.pipeline import Pipeline -from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airflow.py b/ingestion/src/metadata/ingestion/source/pipeline/airflow.py index 03e2988817a..1163437b63b 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/airflow.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/airflow.py @@ -24,7 +24,6 @@ from sqlalchemy.orm import Session from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest from metadata.generated.schema.entity.data.pipeline import ( - Pipeline, PipelineStatus, StatusType, Task, diff --git a/ingestion/src/metadata/ingestion/stage/migrate.py b/ingestion/src/metadata/ingestion/stage/migrate.py index 524c96a4d7f..f9c6dfb611d 100644 --- a/ingestion/src/metadata/ingestion/stage/migrate.py +++ b/ingestion/src/metadata/ingestion/stage/migrate.py @@ -17,7 +17,6 @@ from metadata.config.common import ConfigModel from metadata.generated.schema.entity.data.glossary import Glossary from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm from metadata.generated.schema.entity.data.pipeline import Pipeline -from metadata.generated.schema.entity.data.table import Table from metadata.generated.schema.entity.data.topic import Topic from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( OpenMetadataConnection, diff --git a/ingestion/src/metadata/orm_profiler/processor/orm_profiler.py b/ingestion/src/metadata/orm_profiler/processor/orm_profiler.py index ecb2a927255..c7d5d04ed9d 100644 --- a/ingestion/src/metadata/orm_profiler/processor/orm_profiler.py +++ b/ingestion/src/metadata/orm_profiler/processor/orm_profiler.py @@ -38,7 +38,6 @@ from metadata.ingestion.api.processor import Processor, ProcessorStatus from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.orm_profiler.api.models import ProfilerProcessorConfig, ProfilerResponse from metadata.orm_profiler.interfaces.interface_protocol import InterfaceProtocol -from metadata.orm_profiler.interfaces.sqa_profiler_interface import SQAProfilerInterface from metadata.orm_profiler.metrics.registry import Metrics from metadata.orm_profiler.orm.converter import ometa_to_orm from metadata.orm_profiler.profiler.core import Profiler @@ -47,8 +46,6 @@ from metadata.orm_profiler.profiler.handle_partition import ( get_partition_cols, is_partitioned, ) -from metadata.orm_profiler.profiler.sampler import Sampler -from metadata.orm_profiler.validations.core import validation_enum_registry from metadata.orm_profiler.validations.models import TestDef from metadata.utils.helpers import get_start_and_end diff --git a/ingestion/src/metadata/orm_profiler/validations/table/table_custom_sql_query.py b/ingestion/src/metadata/orm_profiler/validations/table/table_custom_sql_query.py index 776b7edde1b..c90e9fc477e 100644 --- a/ingestion/src/metadata/orm_profiler/validations/table/table_custom_sql_query.py +++ b/ingestion/src/metadata/orm_profiler/validations/table/table_custom_sql_query.py @@ -19,7 +19,6 @@ from datetime import datetime from sqlalchemy import text from sqlalchemy.orm import DeclarativeMeta, Session -from metadata.generated.schema.entity.data.table import TableProfile from metadata.generated.schema.tests.basic import TestCaseResult, TestCaseStatus from metadata.generated.schema.tests.table.tableCustomSQLQuery import ( TableCustomSQLQuery, diff --git a/ingestion/src/metadata/utils/sql_lineage.py b/ingestion/src/metadata/utils/sql_lineage.py index 9dd53d239c6..34185d013df 100644 --- a/ingestion/src/metadata/utils/sql_lineage.py +++ b/ingestion/src/metadata/utils/sql_lineage.py @@ -23,10 +23,8 @@ from metadata.generated.schema.type.entityLineage import ( LineageDetails, ) from metadata.generated.schema.type.entityReference import EntityReference -from metadata.ingestion.ometa.client import APIError from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.utils import fqn -from metadata.utils.helpers import get_formatted_entity_name from metadata.utils.logger import utils_logger from metadata.utils.lru_cache import LRUCache diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ManageTab/ManageTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ManageTab/ManageTab.component.tsx index 8e7a9c36b12..a010de70032 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ManageTab/ManageTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ManageTab/ManageTab.component.tsx @@ -281,7 +281,7 @@ const ManageTab: FunctionComponent = ({ return ( isAdminUser || isAuthDisabled || - userPermissions[Operation.TeamEditUsers] || + userPermissions[Operation.EditUsers] || !hasEditAccess ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/RulesModal/AddRuleModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/RulesModal/AddRuleModal.tsx index 56505727dd3..fd7d644036a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/RulesModal/AddRuleModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/RulesModal/AddRuleModal.tsx @@ -109,9 +109,7 @@ const AddRuleModal: FC = ({ - + {errorData?.operation && errorMsg(errorData.operation)} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/TeamDetails/TeamDetails.tsx b/openmetadata-ui/src/main/resources/ui/src/components/TeamDetails/TeamDetails.tsx index 5ba11448c25..617dba31246 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/TeamDetails/TeamDetails.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/TeamDetails/TeamDetails.tsx @@ -343,7 +343,7 @@ const TeamDetails = ({ ? `as ${teamUsersSearchText}.` : `added yet.`}

- {isActionAllowed(userPermissions[Operation.TeamEditUsers]) ? ( + {isActionAllowed(userPermissions[Operation.EditUsers]) ? ( <>

Would like to start adding some?