Fixes #6336 - Add API to list of operations per resource for policy authoring (#6337)

* Fixes #6336 - Add API to list of operations per resource to be used for writing policies

* Fixing UI build errors

* Include only JSON schema file and not JSON data files in python code generation

* Fix generate

* Update operations list

Co-authored-by: Pere Miquel Brull <peremiquelbrull@gmail.com>
This commit is contained in:
Suresh Srinivas 2022-07-26 12:08:01 -07:00 committed by GitHub
parent 7da74d48e5
commit a23a9c84c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
50 changed files with 528 additions and 156 deletions

View File

@ -60,7 +60,8 @@ py_format_check: ## Check if Python sources are correctly formatted
generate: ## Generate the pydantic models from the JSON Schemas to the ingestion module
@echo "Running Datamodel Code Generator"
@echo "Make sure to first run the install_dev recipe"
datamodel-codegen --input catalog-rest-service/src/main/resources/json --input-file-type jsonschema --output ingestion/src/metadata/generated --set-default-enum-member
mkdir -p ingestion/src/metadata/generated
datamodel-codegen --input catalog-rest-service/src/main/resources/json/schema --input-file-type jsonschema --output ingestion/src/metadata/generated/schema --set-default-enum-member
$(MAKE) py_antlr
$(MAKE) install
@ -162,7 +163,7 @@ core_generate: ## Generate the pydantic models from the JSON Schemas to the ing
$(MAKE) core_install_dev
mkdir -p ingestion-core/src/metadata/generated; \
. ingestion-core/venv/bin/activate; \
datamodel-codegen --input catalog-rest-service/src/main/resources/json --input-file-type jsonschema --output ingestion-core/src/metadata/generated
datamodel-codegen --input catalog-rest-service/src/main/resources/json/schema --input-file-type jsonschema --output ingestion-core/src/metadata/generated/schema
$(MAKE) core_py_antlr
.PHONY: core_bump_version_dev

View File

@ -51,7 +51,7 @@ public final class Entity {
// Canonical entity name to corresponding EntityRepository map
private static final Map<String, EntityRepository<?>> ENTITY_REPOSITORY_MAP = new HashMap<>();
// List of entities
// List of all the entities
private static final List<String> ENTITY_LIST = new ArrayList<>();
// Common field names
@ -152,7 +152,7 @@ public final class Entity {
entityRepository.getClass().getSimpleName());
}
public static List<String> listEntities() {
public static List<String> getEntityList() {
return Collections.unmodifiableList(ENTITY_LIST);
}

View File

@ -0,0 +1,23 @@
package org.openmetadata.catalog;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import org.openmetadata.catalog.type.ResourceDescriptor;
public class ResourceRegistry {
private static final ResourceRegistry registry = new ResourceRegistry();
private static final List<ResourceDescriptor> RESOURCE_DESCRIPTORS = new ArrayList<>();
private ResourceRegistry() {}
public static void add(List<ResourceDescriptor> resourceDescriptors) {
RESOURCE_DESCRIPTORS.addAll(resourceDescriptors);
RESOURCE_DESCRIPTORS.sort(Comparator.comparing(ResourceDescriptor::getName));
}
public static List<ResourceDescriptor> listResourceDescriptors() {
return Collections.unmodifiableList(RESOURCE_DESCRIPTORS);
}
}

View File

@ -13,6 +13,8 @@
package org.openmetadata.catalog.resources.databases;
import static org.openmetadata.catalog.type.MetadataOperation.*;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;

View File

@ -23,8 +23,6 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import javax.json.JsonPatch;
import javax.validation.Valid;
@ -47,8 +45,10 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.apache.maven.shared.utils.io.IOUtil;
import org.openmetadata.catalog.CatalogApplicationConfig;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.ResourceRegistry;
import org.openmetadata.catalog.api.policies.CreatePolicy;
import org.openmetadata.catalog.entity.policies.Policy;
import org.openmetadata.catalog.jdbi3.CollectionDAO;
@ -61,6 +61,9 @@ import org.openmetadata.catalog.security.policyevaluator.PolicyEvaluator;
import org.openmetadata.catalog.type.EntityHistory;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.Include;
import org.openmetadata.catalog.type.ResourceDescriptor;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.ResultList;
@Slf4j
@ -71,7 +74,6 @@ import org.openmetadata.catalog.util.ResultList;
@Collection(name = "policies")
public class PolicyResource extends EntityResource<Policy, PolicyRepository> {
public static final String COLLECTION_PATH = "v1/policies/";
public static final List<String> RESOURCES = new ArrayList<>();
@Override
public Policy addHref(UriInfo uriInfo, Policy policy) {
@ -92,6 +94,24 @@ public class PolicyResource extends EntityResource<Policy, PolicyRepository> {
// Load any existing rules from database, before loading seed data.
policyEvaluator.load();
dao.initSeedDataFromResources();
initResourceDescriptors();
}
/** Initialize list of resources and the corresponding operations */
private void initResourceDescriptors() throws IOException {
List<String> jsonDataFiles = EntityUtil.getJsonDataResources(".*json/data/ResourceDescriptors.json$");
if (jsonDataFiles.size() != 1) {
LOG.warn("Invalid number of jsonDataFiles {}. Only one expected.", jsonDataFiles.size());
return;
}
String jsonDataFile = jsonDataFiles.get(0);
try {
String json = IOUtil.toString(getClass().getClassLoader().getResourceAsStream(jsonDataFile));
List<ResourceDescriptor> resourceDetails = JsonUtils.readObjects(json, ResourceDescriptor.class);
ResourceRegistry.add(resourceDetails);
} catch (Exception e) {
LOG.warn("Failed to initialize the {} from file {}", entityType, jsonDataFile, e);
}
}
public static class PolicyList extends ResultList<Policy> {
@ -105,6 +125,17 @@ public class PolicyResource extends EntityResource<Policy, PolicyRepository> {
}
}
public static class ResourceDescriptorList extends ResultList<ResourceDescriptor> {
@SuppressWarnings("unused")
ResourceDescriptorList() {
// Empty constructor needed for deserialization
}
public ResourceDescriptorList(List<ResourceDescriptor> data) {
super(data, null, null, data.size());
}
}
public static final String FIELDS = "owner,location";
@GET
@ -283,16 +314,9 @@ public class PolicyResource extends EntityResource<Policy, PolicyRepository> {
responseCode = "404",
description = "Policy for instance {id} and version {version} is" + " " + "not found")
})
public List<String> listPolicyResources(@Context UriInfo uriInfo, @Context SecurityContext securityContext)
throws IOException {
if (RESOURCES.isEmpty()) {
// Load set of resource types
RESOURCES.addAll(Entity.listEntities());
RESOURCES.add("lineage");
RESOURCES.add("feed");
Collections.sort(RESOURCES);
}
return RESOURCES;
public ResultList<ResourceDescriptor> listPolicyResources(
@Context UriInfo uriInfo, @Context SecurityContext securityContext) throws IOException {
return new ResourceDescriptorList(ResourceRegistry.listResourceDescriptors());
}
@POST

View File

@ -15,6 +15,7 @@ package org.openmetadata.catalog.security.policyevaluator;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import lombok.NonNull;
@ -55,7 +56,7 @@ import org.openmetadata.catalog.util.JsonUtils;
public class PolicyEvaluator {
private PolicyRepository policyRepository;
private final ConcurrentHashMap<UUID, Rules> policyToRules = new ConcurrentHashMap<>();
private final Map<UUID, Rules> policyToRules = new ConcurrentHashMap<>();
private final RulesEngine checkPermissionRulesEngine;
private final RulesEngine allowedOperationsRulesEngine;

View File

@ -34,7 +34,7 @@ public class JsonPatchUtils {
return MetadataOperation.EDIT_OWNER;
}
if (path.startsWith("/users")) { // Ability to update users within a team.
return MetadataOperation.TEAM_EDIT_USERS;
return MetadataOperation.EDIT_USERS;
}
return null;
}

View File

@ -0,0 +1,361 @@
[
{
"name" : "bot",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name": "chart",
"operations": [
"Create",
"Delete",
"ViewAll",
"ViewUsage",
"EditAll",
"EditDescription",
"EditTags",
"EditOwner",
"EditTier",
"EditCustomFields",
"EditLineage"
]
},
{
"name" : "dashboard",
"operations" : [
"Create",
"Delete",
"ViewAll",
"ViewUsage",
"EditAll",
"EditDescription",
"EditTags",
"EditOwner",
"EditTier",
"EditCustomFields",
"EditLineage"
]
},
{
"name" : "dashboardService",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "database",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "databaseSchema",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "databaseService",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "glossary",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "glossaryTerm",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "ingestionPipeline",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "location",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "messagingService",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "metrics",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "mlmodel",
"operations" : [
"Create",
"Delete",
"ViewAll",
"ViewUsage",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "mlmodelService",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "pipeline",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields",
"EditLineage"
]
},
{
"name" : "pipelineService",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "policy",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "report",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "role",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "storageService",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "table",
"operations" : [
"Create",
"Delete",
"ViewAll",
"ViewUsage",
"ViewTests",
"ViewQueries",
"ViewDataProfile",
"ViewSampleData",
"EditAll",
"EditDescription",
"EditTags",
"EditOwner",
"EditTier",
"EditCustomFields",
"EditTests",
"EditQueries",
"EditDataProfile",
"EditSampleData",
"EditLineage"
]
},
{
"name" : "tag",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "tagCategory",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "team",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields",
"EditUsers"
]
},
{
"name" : "topic",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "type",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "user",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "webhook",
"operations" : [
"Create",
"Delete",
"ViewAll",
"EditAll",
"EditDescription",
"EditCustomFields"
]
},
{
"name" : "events",
"operations" : [
"ViewAll"
]
},
{
"name" : "feed",
"operations" : [
"Create",
"Delete",
"ViewAll"
]
}
]

View File

@ -0,0 +1,51 @@
{
"$id": "https://open-metadata.org/schema/entity/data/policies/accessControl/resourceDescriptor.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ResourceDescriptor",
"description": "Resource descriptor",
"type": "object",
"javaType": "org.openmetadata.catalog.type.ResourceDescriptor",
"definitions": {
"operation": {
"javaType": "org.openmetadata.catalog.type.MetadataOperation",
"description": "This schema defines all possible operations on metadata of entities in OpenMetadata.",
"type": "string",
"enum": [
"Create",
"Delete",
"ViewAll",
"ViewUsage",
"ViewTests",
"ViewQueries",
"ViewDataProfile",
"ViewSampleData",
"EditAll",
"EditDescription",
"EditTags",
"EditOwner",
"EditTier",
"EditCustomFields",
"EditLineage",
"EditReviewers",
"EditTests",
"EditQueries",
"EditDataProfile",
"EditSampleData",
"EditUsers"
]
}
},
"properties": {
"name": {
"description": "Name of the resource. For entity related resources, resource name is same as the entity name. Some resources such as lineage are not entities but are resources.",
"type": "string"
},
"operations": {
"description": "List of operations supported by the resource.",
"type": "array",
"items": {
"$ref": "#/definitions/operation"
}
}
}
}

View File

@ -5,36 +5,6 @@
"description": "Describes an Access Control Rule for OpenMetadata Metadata Operations. All non-null user (subject) and entity (object) attributes are evaluated with logical AND.",
"type": "object",
"javaType": "org.openmetadata.catalog.entity.policies.accessControl.Rule",
"definitions": {
"operation": {
"javaType": "org.openmetadata.catalog.type.MetadataOperation",
"description": "This schema defines all possible operations on metadata of data entities.",
"type": "string",
"enum": [
"Create",
"Delete",
"ViewAll",
"ViewUsage",
"ViewTests",
"TableViewQueries",
"TableViewDataProfile",
"TableViewSampleData",
"EditAll",
"EditDescription",
"EditTags",
"EditOwner",
"EditTier",
"EditCustomFields",
"EditLineage",
"EditReviewers",
"EditTests",
"TableEditQueries",
"TableEditDataProfile",
"TableEditSampleData",
"TeamEditUsers"
]
}
},
"properties": {
"name": {
"description": "Name for this Rule.",
@ -56,7 +26,7 @@
},
"operation": {
"description": "Operation on the entity.",
"$ref": "#/definitions/operation",
"$ref": "resourceDescriptor.json#/definitions/operation",
"default": null
},
"allow": {

View File

@ -65,7 +65,7 @@
},
"queryTag": {
"title": "Query Tag",
"description": "Session query tag used to monitor usage on snoflake. To use a query tag snowflake user should have enough privileges to alter the session.",
"description": "Session query tag used to monitor usage on snowflake. To use a query tag snowflake user should have enough privileges to alter the session.",
"type": "string"
},
"privateKey": {

View File

@ -11,7 +11,7 @@
"type": "string"
},
"ordered": {
"description": "Wether or not to considered the order of the list when performing the match",
"description": "Whether or not to considered the order of the list when performing the match",
"type": "boolean",
"default": false
}

View File

@ -2,7 +2,7 @@
"$id": "https://open-metadata.org/schema/entity/data/usageRequest.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Usage Request",
"description": "This schema defines type of table usage request used to publish the usage count on a perticular date",
"description": "This schema defines type of table usage request used to publish the usage count on a particular date",
"javaType": "org.openmetadata.catalog.type.UsageRequest",
"type": "object",
"properties": {

View File

@ -94,14 +94,14 @@ class PermissionsResourceTest extends CatalogApplicationTest {
dataStewardPermissions.put(MetadataOperation.EDIT_OWNER, Boolean.TRUE);
dataStewardPermissions.put(MetadataOperation.EDIT_TAGS, Boolean.TRUE);
// put(MetadataOperation.DecryptTokens, Boolean.FALSE);
dataStewardPermissions.put(MetadataOperation.TEAM_EDIT_USERS, Boolean.FALSE);
dataStewardPermissions.put(MetadataOperation.EDIT_USERS, Boolean.FALSE);
dataConsumerPermissions.put(MetadataOperation.EDIT_DESCRIPTION, Boolean.TRUE);
dataConsumerPermissions.put(MetadataOperation.EDIT_LINEAGE, Boolean.FALSE);
dataConsumerPermissions.put(MetadataOperation.EDIT_OWNER, Boolean.TRUE);
dataConsumerPermissions.put(MetadataOperation.EDIT_TAGS, Boolean.TRUE);
// put(MetadataOperation.DecryptTokens, Boolean.FALSE);
dataConsumerPermissions.put(MetadataOperation.TEAM_EDIT_USERS, Boolean.FALSE);
dataConsumerPermissions.put(MetadataOperation.EDIT_USERS, Boolean.FALSE);
return Stream.of(
Arguments.of(TestUtils.ADMIN_USER_NAME, adminPermissions),

View File

@ -15,7 +15,7 @@ package org.openmetadata.catalog.resources.policies;
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.openmetadata.catalog.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.assertListNotNull;
@ -47,11 +47,13 @@ import org.openmetadata.catalog.exception.CatalogExceptionMessage;
import org.openmetadata.catalog.resources.EntityResourceTest;
import org.openmetadata.catalog.resources.locations.LocationResourceTest;
import org.openmetadata.catalog.resources.policies.PolicyResource.PolicyList;
import org.openmetadata.catalog.resources.policies.PolicyResource.ResourceDescriptorList;
import org.openmetadata.catalog.type.ChangeDescription;
import org.openmetadata.catalog.type.EntityReference;
import org.openmetadata.catalog.type.FieldChange;
import org.openmetadata.catalog.type.MetadataOperation;
import org.openmetadata.catalog.type.PolicyType;
import org.openmetadata.catalog.type.ResourceDescriptor;
import org.openmetadata.catalog.util.EntityUtil;
import org.openmetadata.catalog.util.JsonUtils;
import org.openmetadata.catalog.util.PolicyUtils;
@ -189,10 +191,17 @@ public class PolicyResourceTest extends EntityResourceTest<Policy, CreatePolicy>
@Test
void get_policyResources() throws HttpResponseException {
// Get list of policy resources and make sure it has all the entities and other resources
List<String> resources = getPolicyResources(ADMIN_AUTH_HEADERS);
List<String> entities = Entity.listEntities();
assertTrue(resources.containsAll(entities));
assertTrue(resources.contains("lineage"));
ResourceDescriptorList actualResourceDescriptors = getPolicyResources(ADMIN_AUTH_HEADERS);
assertNotNull(actualResourceDescriptors.getData());
System.out.println(actualResourceDescriptors.getData());
// Ensure all entities are captured in resource descriptor list
List<String> entities = Entity.getEntityList();
for (String entity : entities) {
ResourceDescriptor resourceDescriptor =
actualResourceDescriptors.getData().stream().filter(rd -> rd.getName().equals(entity)).findFirst().get();
assertNotNull(resourceDescriptor);
}
}
@Override
@ -231,8 +240,8 @@ public class PolicyResourceTest extends EntityResourceTest<Policy, CreatePolicy>
return TestUtils.post(getResource("locations"), createLocation, Location.class, ADMIN_AUTH_HEADERS);
}
public final List<String> getPolicyResources(Map<String, String> authHeaders) throws HttpResponseException {
public final ResourceDescriptorList getPolicyResources(Map<String, String> authHeaders) throws HttpResponseException {
WebTarget target = getResource(collectionName + "/resources");
return (List<String>) TestUtils.get(target, List.class, authHeaders);
return TestUtils.get(target, ResourceDescriptorList.class, authHeaders);
}
}

View File

@ -215,7 +215,7 @@ public class TeamResourceTest extends EntityResourceTest<Team, CreateTeam> {
patchEntity(
team.getId(), originalJson, team, SecurityUtil.authHeaders(randomUserName + "@open-metadata.org")),
FORBIDDEN,
CatalogExceptionMessage.noPermission(randomUserName, "TeamEditUsers"));
CatalogExceptionMessage.noPermission(randomUserName, "EditUsers"));
// Ensure user with UpdateTeam permission can add users to a team.
User teamManagerUser = createTeamManager(test);
@ -459,10 +459,7 @@ public class TeamResourceTest extends EntityResourceTest<Team, CreateTeam> {
private User createTeamManager(TestInfo testInfo) throws HttpResponseException {
// Create a rule that can update team
Rule rule =
new Rule()
.withName("TeamManagerPolicy-UpdateTeam")
.withAllow(true)
.withOperation(MetadataOperation.TEAM_EDIT_USERS);
new Rule().withName("TeamManagerPolicy-UpdateTeam").withAllow(true).withOperation(MetadataOperation.EDIT_USERS);
// Create a policy with the rule
PolicyResourceTest policyResourceTest = new PolicyResourceTest();

View File

@ -7,5 +7,5 @@ Provides metadata version information.
from incremental import Version
__version__ = Version("metadata", 0, 12, 0, dev=2)
__version__ = Version("metadata", 0, 12, 0, dev=3)
__all__ = ["__version__"]

View File

@ -24,7 +24,6 @@ from metadata.cli.backup import run_backup
from metadata.cli.docker import run_docker
from metadata.cli.ingest import run_ingest
from metadata.config.common import load_config_file
from metadata.ingestion.api.workflow import Workflow
from metadata.orm_profiler.api.workflow import ProfilerWorkflow
from metadata.utils.logger import cli_logger, set_loggers_level

View File

@ -19,14 +19,6 @@ from metadata.config.workflow import get_source_dir
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection,
)
from metadata.generated.schema.entity.services.dashboardService import (
DashboardConnection,
)
from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
from metadata.generated.schema.entity.services.messagingService import (
MessagingConnection,
)
from metadata.generated.schema.entity.services.metadataService import MetadataConnection
from metadata.generated.schema.metadataIngestion.workflow import (
OpenMetadataWorkflowConfig,
)

View File

@ -14,7 +14,6 @@ from typing import List, Optional
from pydantic import BaseModel
from metadata.generated.schema.entity.data.chart import Chart
from metadata.generated.schema.entity.data.mlmodel import (
MlFeature,
MlHyperParameter,

View File

@ -19,7 +19,6 @@ from typing import Any, Dict, Generic, Optional, Type, TypeVar, Union
from pydantic import BaseModel
from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
from metadata.generated.schema.entity.data.table import Table
from metadata.ingestion.ometa.client import REST, APIError
from metadata.ingestion.ometa.utils import get_entity_type, ometa_logger

View File

@ -20,13 +20,12 @@ from elasticsearch import Elasticsearch
from elasticsearch.connection import create_ssl_context
from metadata.config.common import ConfigModel
from metadata.generated.schema.entity.data.chart import Chart
from metadata.generated.schema.entity.data.dashboard import Dashboard
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
from metadata.generated.schema.entity.data.mlmodel import MlModel
from metadata.generated.schema.entity.data.pipeline import Pipeline, Task
from metadata.generated.schema.entity.data.pipeline import Pipeline
from metadata.generated.schema.entity.data.table import Column, Table
from metadata.generated.schema.entity.data.topic import Topic
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (

View File

@ -41,17 +41,12 @@ from metadata.generated.schema.metadataIngestion.workflow import (
from metadata.generated.schema.type.entityLineage import EntitiesEdge
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.api.source import InvalidSourceException
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
from metadata.ingestion.source.database.common_db_source import SQLSourceStatus
from metadata.utils import fqn
from metadata.utils.connections import get_connection
from metadata.utils.filters import filter_by_chart
from metadata.utils.helpers import (
get_chart_entities_from_id,
get_standard_chart_type,
replace_special_with,
)
from metadata.utils.helpers import get_standard_chart_type, replace_special_with
from metadata.utils.logger import ingestion_logger
from metadata.utils.sql_lineage import search_table_entities

View File

@ -42,7 +42,7 @@ from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.api.source import InvalidSourceException, SourceStatus
from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
from metadata.utils import fqn
from metadata.utils.helpers import get_chart_entities_from_id, get_standard_chart_type
from metadata.utils.helpers import get_standard_chart_type
from metadata.utils.logger import ingestion_logger
logger = ingestion_logger()

View File

@ -40,7 +40,6 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata
OpenMetadataConnection,
)
from metadata.generated.schema.entity.tags.tagCategory import Tag
from metadata.generated.schema.entity.teams.user import User
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
@ -52,7 +51,7 @@ from metadata.ingestion.models.ometa_tag_category import OMetaTagAndCategory
from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
from metadata.utils import fqn
from metadata.utils.filters import filter_by_chart
from metadata.utils.helpers import get_chart_entities_from_id, get_standard_chart_type
from metadata.utils.helpers import get_standard_chart_type
from metadata.utils.logger import ingestion_logger
logger = ingestion_logger()

View File

@ -33,7 +33,6 @@ from metadata.generated.schema.entity.data.table import Table, TableType
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection,
)
from metadata.generated.schema.entity.tags.tagCategory import Tag
from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import (
DatabaseServiceMetadataPipeline,
)
@ -41,12 +40,6 @@ from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.generated.schema.type.tagLabel import (
LabelType,
State,
TagLabel,
TagSource,
)
from metadata.ingestion.models.ometa_tag_category import OMetaTagAndCategory
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.database.database_service import (
@ -57,7 +50,7 @@ from metadata.ingestion.source.database.sql_column_handler import SqlColumnHandl
from metadata.ingestion.source.database.sqlalchemy_source import SqlAlchemySource
from metadata.utils import fqn
from metadata.utils.connections import get_connection, test_connection
from metadata.utils.filters import filter_by_database, filter_by_schema, filter_by_table
from metadata.utils.filters import filter_by_schema, filter_by_table
from metadata.utils.logger import ingestion_logger
from metadata.utils.sql_lineage import (
get_lineage_by_query,

View File

@ -44,7 +44,7 @@ from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.generated.schema.type import basic
from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
from metadata.generated.schema.type.basic import FullyQualifiedEntityName
from metadata.generated.schema.type.tagLabel import (
LabelType,
State,

View File

@ -43,7 +43,6 @@ from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.api.common import Entity
from metadata.ingestion.api.source import InvalidSourceException, SourceStatus
from metadata.ingestion.models.ometa_tag_category import OMetaTagAndCategory
from metadata.ingestion.ometa.ometa_api import OpenMetadata

View File

@ -16,7 +16,6 @@ from typing import Iterable, List, Optional
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
from metadata.generated.schema.entity.data.location import Location, LocationType
from metadata.generated.schema.entity.data.pipeline import Task
from metadata.generated.schema.entity.data.table import Column, Table, TableType
from metadata.generated.schema.entity.services.connections.database.glueConnection import (
GlueConnection,

View File

@ -25,7 +25,6 @@ from metadata.generated.schema.metadataIngestion.workflow import (
)
from metadata.ingestion.api.source import InvalidSourceException
from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
from metadata.utils.connections import get_connection
from metadata.utils.filters import filter_by_database
from metadata.utils.logger import ingestion_logger

View File

@ -13,7 +13,6 @@
from sqlalchemy.engine.reflection import Inspector
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.services.connections.database.oracleConnection import (
OracleConnection,
)
@ -23,7 +22,6 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.api.source import InvalidSourceException
from metadata.ingestion.source.database.common_db_source import CommonDbSourceService

View File

@ -15,7 +15,6 @@ from typing import Iterable
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.inspection import inspect
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
PostgresConnection,
)
@ -27,10 +26,8 @@ from metadata.generated.schema.entity.services.connections.metadata.openMetadata
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.api.source import InvalidSourceException, SourceStatus
from metadata.ingestion.api.source import InvalidSourceException
from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
from metadata.utils.connections import get_connection
from metadata.utils.filters import filter_by_database
from metadata.utils.logger import ingestion_logger

View File

@ -18,9 +18,6 @@ from typing import Iterator, Union
from metadata.generated.schema.entity.services.connections.database.redshiftConnection import (
RedshiftConnection,
)
from metadata.generated.schema.entity.services.databaseService import (
DatabaseServiceType,
)
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)

View File

@ -26,7 +26,6 @@ from metadata.generated.schema.entity.services.databaseService import (
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.generated.schema.metadataIngestion.workflow import WorkflowConfig
from metadata.generated.schema.type.tableQuery import TableQuery
from metadata.ingestion.api.source import InvalidSourceException

View File

@ -17,14 +17,13 @@ from typing import List, Optional, Set, Tuple
from sqlalchemy.engine import Engine
from sqlalchemy.engine.reflection import Inspector
from metadata.generated.schema.entity.data.table import Column, DataModel, Table
from metadata.generated.schema.entity.data.table import Column, DataModel
from metadata.generated.schema.metadataIngestion.databaseServiceMetadataPipeline import (
DatabaseServiceMetadataPipeline,
)
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.generated.schema.type.tagLabel import TagLabel
from metadata.ingestion.models.topology import TopologyContext
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.database.database_service import SQLSourceStatus

View File

@ -18,13 +18,8 @@ from typing import Any, Iterable, List, Optional
from confluent_kafka.admin import AdminClient, ConfigResource
from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest
from metadata.generated.schema.api.data.createTopic import CreateTopicRequest
from metadata.generated.schema.entity.data.topic import (
SchemaType,
Topic,
TopicSampleData,
)
from metadata.generated.schema.entity.data.topic import Topic
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection,
)
@ -38,7 +33,7 @@ from metadata.generated.schema.metadataIngestion.messagingServiceMetadataPipelin
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)
from metadata.ingestion.api.source import InvalidSourceException, Source, SourceStatus
from metadata.ingestion.api.source import Source, SourceStatus
from metadata.ingestion.api.topology_runner import TopologyRunnerMixin
from metadata.ingestion.models.topology import (
NodeStage,

View File

@ -50,15 +50,10 @@ from metadata.ingestion.api.source import InvalidSourceException, Source, Source
from metadata.ingestion.models.ometa_table_db import OMetaDatabaseAndTable
from metadata.ingestion.models.ometa_tag_category import OMetaTagAndCategory
from metadata.ingestion.models.user import OMetaUserProfile
from metadata.ingestion.ometa.client import APIError
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.utils import fqn
from metadata.utils.column_type_parser import ColumnTypeParser
from metadata.utils.helpers import (
get_chart_entities_from_id,
get_dashboard_service_or_create,
get_standard_chart_type,
)
from metadata.utils.helpers import get_chart_entities_from_id, get_standard_chart_type
from metadata.utils.logger import ingestion_logger
from metadata.utils.neo4j_helper import Neo4JConfig, Neo4jHelper
from metadata.utils.sql_queries import (

View File

@ -8,12 +8,8 @@ from typing import Any, Dict, Iterable, List
import yaml
from importlib_metadata import SelectableGroups
from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
from metadata.generated.schema.api.data.createTopic import CreateTopicRequest
from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
from metadata.generated.schema.api.services.createDatabaseService import (
CreateDatabaseServiceRequest,
)
from metadata.generated.schema.entity.data.database import Database
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
from metadata.generated.schema.entity.data.pipeline import Pipeline
@ -24,12 +20,8 @@ from metadata.generated.schema.entity.services.connections.metadata.atlasConnect
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection,
)
from metadata.generated.schema.entity.services.databaseService import (
DatabaseService,
DatabaseServiceType,
)
from metadata.generated.schema.entity.services.databaseService import DatabaseService
from metadata.generated.schema.entity.services.messagingService import MessagingService
from metadata.generated.schema.entity.services.metadataService import MetadataService
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
)

View File

@ -17,7 +17,6 @@ from metadata.generated.schema.entity.data.dashboard import Dashboard
from metadata.generated.schema.entity.data.glossary import Glossary
from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
from metadata.generated.schema.entity.data.pipeline import Pipeline
from metadata.generated.schema.entity.data.table import Table
from metadata.generated.schema.entity.data.topic import Topic
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection,

View File

@ -24,7 +24,6 @@ from sqlalchemy.orm import Session
from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest
from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
from metadata.generated.schema.entity.data.pipeline import (
Pipeline,
PipelineStatus,
StatusType,
Task,

View File

@ -17,7 +17,6 @@ from metadata.config.common import ConfigModel
from metadata.generated.schema.entity.data.glossary import Glossary
from metadata.generated.schema.entity.data.glossaryTerm import GlossaryTerm
from metadata.generated.schema.entity.data.pipeline import Pipeline
from metadata.generated.schema.entity.data.table import Table
from metadata.generated.schema.entity.data.topic import Topic
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection,

View File

@ -38,7 +38,6 @@ from metadata.ingestion.api.processor import Processor, ProcessorStatus
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.orm_profiler.api.models import ProfilerProcessorConfig, ProfilerResponse
from metadata.orm_profiler.interfaces.interface_protocol import InterfaceProtocol
from metadata.orm_profiler.interfaces.sqa_profiler_interface import SQAProfilerInterface
from metadata.orm_profiler.metrics.registry import Metrics
from metadata.orm_profiler.orm.converter import ometa_to_orm
from metadata.orm_profiler.profiler.core import Profiler
@ -47,8 +46,6 @@ from metadata.orm_profiler.profiler.handle_partition import (
get_partition_cols,
is_partitioned,
)
from metadata.orm_profiler.profiler.sampler import Sampler
from metadata.orm_profiler.validations.core import validation_enum_registry
from metadata.orm_profiler.validations.models import TestDef
from metadata.utils.helpers import get_start_and_end

View File

@ -19,7 +19,6 @@ from datetime import datetime
from sqlalchemy import text
from sqlalchemy.orm import DeclarativeMeta, Session
from metadata.generated.schema.entity.data.table import TableProfile
from metadata.generated.schema.tests.basic import TestCaseResult, TestCaseStatus
from metadata.generated.schema.tests.table.tableCustomSQLQuery import (
TableCustomSQLQuery,

View File

@ -23,10 +23,8 @@ from metadata.generated.schema.type.entityLineage import (
LineageDetails,
)
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.ometa.client import APIError
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.utils import fqn
from metadata.utils.helpers import get_formatted_entity_name
from metadata.utils.logger import utils_logger
from metadata.utils.lru_cache import LRUCache

View File

@ -281,7 +281,7 @@ const ManageTab: FunctionComponent<ManageProps> = ({
return (
isAdminUser ||
isAuthDisabled ||
userPermissions[Operation.TeamEditUsers] ||
userPermissions[Operation.EditUsers] ||
!hasEditAccess
);
};

View File

@ -109,9 +109,7 @@ const AddRuleModal: FC<AddRuleProps> = ({
<option value={Operation.EditLineage}>Edit Lineage</option>
<option value={Operation.EditOwner}>Edit Owner</option>
<option value={Operation.EditTags}>Edit Tags</option>
<option value={Operation.TeamEditUsers}>
Edit Team Users
</option>
<option value={Operation.EditUsers}>Edit Team Users</option>
</select>
{errorData?.operation && errorMsg(errorData.operation)}
</div>

View File

@ -343,7 +343,7 @@ const TeamDetails = ({
? `as ${teamUsersSearchText}.`
: `added yet.`}
</p>
{isActionAllowed(userPermissions[Operation.TeamEditUsers]) ? (
{isActionAllowed(userPermissions[Operation.EditUsers]) ? (
<>
<p>Would like to start adding some?</p>
<Button

View File

@ -574,7 +574,7 @@ declare module 'Models' {
EditDescription: boolean;
EditLineage: boolean;
EditTags: boolean;
TeamEditUsers: boolean;
EditUsers: boolean;
}
export interface EditorContentRef {
getEditorContent: () => string;

View File

@ -230,7 +230,7 @@ const UserCard = ({
<NonAdminAction
html={<>You do not have permission to update the team.</>}
isOwner={isOwner}
permission={Operation.TeamEditUsers}
permission={Operation.EditUsers}
position="bottom">
<span
className={classNames('tw-h-8 tw-rounded tw-mb-3', {
@ -238,7 +238,7 @@ const UserCard = ({
!isAdminUser &&
!isAuthDisabled &&
!isOwner &&
!userPermissions[Operation.TeamEditUsers],
!userPermissions[Operation.EditUsers],
})}
data-testid="remove"
onClick={() => onRemove?.(item.id as string)}>

View File

@ -445,7 +445,7 @@ const TeamsPage = () => {
{isAdminUser ||
isAuthDisabled ||
isOwner() ||
userPermissions[Operation.TeamEditUsers] ? (
userPermissions[Operation.EditUsers] ? (
<>
<p>Would like to start adding some?</p>
<Button
@ -780,7 +780,7 @@ const TeamsPage = () => {
</Fragment>
}
isOwner={isOwner()}
permission={Operation.TeamEditUsers}
permission={Operation.EditUsers}
position="bottom">
<Button
className={classNames(
@ -789,7 +789,7 @@ const TeamsPage = () => {
'tw-opacity-40':
!isAdminUser &&
!isAuthDisabled &&
!userPermissions[Operation.TeamEditUsers] &&
!userPermissions[Operation.EditUsers] &&
!isOwner(),
}
)}