From a1b9d3fe65a6c8d151e33a2e2dfe6d2d46dc5575 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 30 Jan 2023 21:34:34 -0800 Subject: [PATCH] Fixes 10021 - Add CSV import/export for users (#10022) * Fixes 10021 - Add CSV import/export for users * Fixes 10021 - Add CSV import/export for users --- .../openmetadata/common/utils/CommonUtil.java | 5 +- .../java/org/openmetadata/csv/CsvUtil.java | 4 +- .../java/org/openmetadata/csv/EntityCsv.java | 4 +- .../java/org/openmetadata/service/Entity.java | 12 +- .../alerts/AlertsPublisherManager.java | 6 +- .../emailAlert/EmailAlertPublisher.java | 4 +- .../service/jdbi3/AlertRepository.java | 2 +- .../service/jdbi3/GlossaryRepository.java | 3 +- .../service/jdbi3/KpiRepository.java | 3 +- .../service/jdbi3/TeamRepository.java | 13 +- .../service/jdbi3/UserRepository.java | 159 ++++++++++++++++-- .../resources/alerts/AlertResource.java | 7 +- .../permissions/PermissionsResource.java | 6 +- .../service/resources/tags/TagLabelCache.java | 23 +-- .../service/resources/tags/TagResource.java | 8 +- .../service/resources/teams/RoleResource.java | 6 +- .../service/resources/teams/TeamResource.java | 5 +- .../service/resources/teams/UserResource.java | 75 +++++++++ .../secrets/SecretsManagerUpdateService.java | 12 +- .../service/security/NoopAuthorizer.java | 4 +- .../security/policyevaluator/PolicyCache.java | 6 +- .../security/policyevaluator/RoleCache.java | 6 +- .../policyevaluator/SubjectCache.java | 11 +- .../TestCaseResourceContext.java | 5 +- .../service/util/ElasticSearchIndexUtil.java | 6 +- .../service/util/NotificationHandler.java | 4 +- .../openmetadata/service/util/UserUtil.java | 4 +- .../json/data/user/userCsvDocumentation.json | 76 +++++++++ .../org/openmetadata/csv/CsvUtilTest.java | 3 +- .../org/openmetadata/csv/EntityCsvTest.java | 2 +- .../service/resources/EntityResourceTest.java | 4 +- .../resources/teams/TeamResourceTest.java | 8 + .../resources/teams/UserResourceTest.java | 115 +++++++++++-- .../resources/json/schema/type/basic.json | 5 + 34 files changed, 509 insertions(+), 107 deletions(-) create mode 100644 openmetadata-service/src/main/resources/json/data/user/userCsvDocumentation.json diff --git a/common/src/main/java/org/openmetadata/common/utils/CommonUtil.java b/common/src/main/java/org/openmetadata/common/utils/CommonUtil.java index 7096c767ae5..25121a84e28 100644 --- a/common/src/main/java/org/openmetadata/common/utils/CommonUtil.java +++ b/common/src/main/java/org/openmetadata/common/utils/CommonUtil.java @@ -24,6 +24,7 @@ import java.nio.file.Paths; import java.text.DateFormat; import java.text.ParseException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Base64; import java.util.Calendar; import java.util.Collection; @@ -170,12 +171,12 @@ public final class CommonUtil { return IOUtils.toString(Objects.requireNonNull(loader.getResourceAsStream(file)), UTF_8); } - /** Return list of entiries that are modifiable for performing sort and other operations */ + /** Return list of entries that are modifiable for performing sort and other operations */ @SafeVarargs public static List listOf(T... entries) { if (entries == null) { return Collections.emptyList(); } - return new ArrayList<>(List.of(entries)); + return new ArrayList<>(Arrays.asList(entries)); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java index d877730db77..d165cdde10b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java @@ -97,9 +97,7 @@ public final class CsvUtil { } public static List addField(List record, Boolean field) { - if (field != null) { - record.add(field.toString()); - } + record.add(field == null ? "" : field.toString()); return record; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 485eac17ba4..f1117f97afc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -305,12 +305,12 @@ public abstract class EntityCsv { entity.setId(UUID.randomUUID()); entity.setUpdatedBy(importedBy); entity.setUpdatedAt(System.currentTimeMillis()); - EntityRepository repository = Entity.getEntityRepository(entityType); + EntityRepository repository = (EntityRepository) Entity.getEntityRepository(entityType); Response.Status responseStatus; if (!importResult.getDryRun()) { try { repository.prepareInternal(entity); - PutResponse response = repository.createOrUpdate(null, entity); + PutResponse response = repository.createOrUpdate(null, entity); responseStatus = response.getStatus(); } catch (Exception ex) { importFailure(resultsPrinter, ex.getMessage(), record); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index b231c7a87bf..cd30099ad66 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -48,7 +48,7 @@ public final class Entity { private static final Map> DAO_MAP = new HashMap<>(); // Canonical entity name to corresponding EntityRepository map - private static final Map> ENTITY_REPOSITORY_MAP = new HashMap<>(); + private static final Map> ENTITY_REPOSITORY_MAP = new HashMap<>(); // List of all the entities private static final List ENTITY_LIST = new ArrayList<>(); @@ -258,10 +258,14 @@ public final class Entity { return entity; } - /** Retrieve the corresponding entity repository for a given entity name. */ - public static EntityRepository getEntityRepository(@NonNull String entityType) { + /** + * Retrieve the corresponding entity repository for a given entity name. + * + * @return + */ + public static EntityRepository getEntityRepository(@NonNull String entityType) { @SuppressWarnings("unchecked") - EntityRepository entityRepository = (EntityRepository) ENTITY_REPOSITORY_MAP.get(entityType); + EntityRepository entityRepository = ENTITY_REPOSITORY_MAP.get(entityType); if (entityRepository == null) { throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(entityType)); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/alerts/AlertsPublisherManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/alerts/AlertsPublisherManager.java index c7ac909072a..d47d0c932aa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/alerts/AlertsPublisherManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/alerts/AlertsPublisherManager.java @@ -18,8 +18,8 @@ import org.openmetadata.schema.entity.alerts.AlertActionStatus; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.Entity; import org.openmetadata.service.events.EventPubSub; +import org.openmetadata.service.jdbi3.AlertActionRepository; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.EntityRepository; @Slf4j public class AlertsPublisherManager { @@ -47,10 +47,10 @@ public class AlertsPublisherManager { } public void addAlertActionPublishers(Alert alert) throws IOException { - EntityRepository alertActionEntityRepository = Entity.getEntityRepository(ALERT_ACTION); + AlertActionRepository alertActionRepository = (AlertActionRepository) Entity.getEntityRepository(ALERT_ACTION); for (EntityReference alertActionRef : alert.getAlertActions()) { AlertAction action = - alertActionEntityRepository.get(null, alertActionRef.getId(), alertActionEntityRepository.getFields("*")); + alertActionRepository.get(null, alertActionRef.getId(), alertActionRepository.getFields("*")); addAlertActionPublisher(alert, action); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/alerts/emailAlert/EmailAlertPublisher.java b/openmetadata-service/src/main/java/org/openmetadata/service/alerts/emailAlert/EmailAlertPublisher.java index 8fb3483dcad..9937a5ef651 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/alerts/emailAlert/EmailAlertPublisher.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/alerts/emailAlert/EmailAlertPublisher.java @@ -20,8 +20,8 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.alerts.AlertsActionPublisher; import org.openmetadata.service.events.errors.EventPublisherException; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.jdbi3.UserRepository; import org.openmetadata.service.resources.events.EventResource; import org.openmetadata.service.security.policyevaluator.SubjectCache; import org.openmetadata.service.util.ChangeEventParser; @@ -74,7 +74,7 @@ public class EmailAlertPublisher extends AlertsActionPublisher { private Set sendToAdmins() { Set emailList = new HashSet<>(); - EntityRepository userEntityRepository = Entity.getEntityRepository(USER); + UserRepository userEntityRepository = (UserRepository) Entity.getEntityRepository(USER); ResultList result; ListFilter listFilter = new ListFilter(Include.ALL); listFilter.addQueryParam("isAdmin", "true"); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AlertRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AlertRepository.java index bac791bbf48..df0f3731f53 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AlertRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AlertRepository.java @@ -102,7 +102,7 @@ public class AlertRepository extends EntityRepository { List alertActionList = new ArrayList<>(); List records = daoCollection.relationshipDAO().findTo(alertId.toString(), ALERT, CONTAINS.ordinal(), ALERT_ACTION); - EntityRepository alertEntityRepository = Entity.getEntityRepository(ALERT_ACTION); + AlertActionRepository alertEntityRepository = (AlertActionRepository) Entity.getEntityRepository(ALERT_ACTION); for (CollectionDAO.EntityRelationshipRecord record : records) { AlertAction alertAction = alertEntityRepository.get(null, record.getId(), alertEntityRepository.getFields("*")); alertAction.setStatusDetails(getActionStatus(alertId, alertAction.getId())); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java index 5692d0fcd0f..34a4e10e4f8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java @@ -124,10 +124,11 @@ public class GlossaryRepository extends EntityRepository { return new GlossaryUpdater(original, updated, operation); } + /** Export glossary as CSV */ @Override public String exportToCsv(String name, String user) throws IOException { Glossary glossary = getByName(null, name, Fields.EMPTY_FIELDS); // Validate glossary name - EntityRepository repository = Entity.getEntityRepository(Entity.GLOSSARY_TERM); + GlossaryTermRepository repository = (GlossaryTermRepository) Entity.getEntityRepository(Entity.GLOSSARY_TERM); ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parent", name); List terms = repository.listAll(repository.getFields("reviewers,tags,relatedTerms"), filter); terms.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName)); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/KpiRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/KpiRepository.java index d4ccaa4dd96..85955fb6000 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/KpiRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/KpiRepository.java @@ -55,7 +55,8 @@ public class KpiRepository extends EntityRepository { public void prepare(Kpi kpi) throws IOException { // validate targetDefinition Entity.getEntityReferenceById(Entity.DATA_INSIGHT_CHART, kpi.getDataInsightChart().getId(), Include.NON_DELETED); - EntityRepository dataInsightChartRepository = Entity.getEntityRepository(DATA_INSIGHT_CHART); + DataInsightChartRepository dataInsightChartRepository = + (DataInsightChartRepository) Entity.getEntityRepository(DATA_INSIGHT_CHART); DataInsightChart chart = dataInsightChartRepository.get( null, kpi.getDataInsightChart().getId(), dataInsightChartRepository.getFields("metrics")); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java index 5c1505d4b10..4ee87293c41 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java @@ -194,17 +194,15 @@ public class TeamRepository extends EntityRepository { SubjectCache.getInstance().invalidateTeam(team.getId()); } - /** Export team as CSV */ @Override public String exportToCsv(String parentTeam, String user) throws IOException { - Team team = getByName(null, parentTeam, Fields.EMPTY_FIELDS); // Validate glossary name - return new TeamCsv(team, user).exportCsv(this); + Team team = getByName(null, parentTeam, Fields.EMPTY_FIELDS); // Validate team name + return new TeamCsv(team, user).exportCsv(); } - /** Load CSV provided for bulk upload */ @Override public CsvImportResult importFromCsv(String name, String csv, boolean dryRun, String user) throws IOException { - Team team = getByName(null, name, Fields.EMPTY_FIELDS); // Validate glossary name + Team team = getByName(null, name, Fields.EMPTY_FIELDS); // Validate team name TeamCsv teamCsv = new TeamCsv(team, user); return teamCsv.importCsv(csv, dryRun); } @@ -610,7 +608,7 @@ public class TeamRepository extends EntityRepository { throws IOException { // Export the entire hierarchy of teams final ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parentTeam", parentTeam); - List list = repository.listAfter(null, fields, filter, 10000, null).getData(); + List list = repository.listAll(fields, filter); if (nullOrEmpty(list)) { return teams; } @@ -621,7 +619,8 @@ public class TeamRepository extends EntityRepository { return teams; } - public String exportCsv(TeamRepository repository) throws IOException { + public String exportCsv() throws IOException { + TeamRepository repository = (TeamRepository) Entity.getEntityRepository(TEAM); final Fields fields = repository.getFields("owner,defaultRoles,parents,policies"); return exportCsv(listTeams(repository, team.getName(), new ArrayList<>(), fields)); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java index 6fc150d915f..5c26f6f9663 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java @@ -13,8 +13,14 @@ package org.openmetadata.service.jdbi3; +import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.csv.CsvUtil.addEntityReferences; +import static org.openmetadata.csv.CsvUtil.addField; +import static org.openmetadata.service.Entity.ROLE; import static org.openmetadata.service.Entity.TEAM; +import static org.openmetadata.service.Entity.USER; import java.io.IOException; import java.util.ArrayList; @@ -26,6 +32,9 @@ import java.util.UUID; import java.util.stream.Collectors; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.CSVRecord; +import org.openmetadata.csv.EntityCsv; import org.openmetadata.schema.api.teams.CreateTeam.TeamType; import org.openmetadata.schema.auth.SSOAuthMechanism; import org.openmetadata.schema.entity.teams.AuthenticationMechanism; @@ -34,6 +43,10 @@ import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.csv.CsvDocumentation; +import org.openmetadata.schema.type.csv.CsvErrorType; +import org.openmetadata.schema.type.csv.CsvHeader; +import org.openmetadata.schema.type.csv.CsvImportResult; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.CatalogExceptionMessage; @@ -55,14 +68,7 @@ public class UserRepository extends EntityRepository { private final EntityReference organization; public UserRepository(CollectionDAO dao) { - super( - UserResource.COLLECTION_PATH, - Entity.USER, - User.class, - dao.userDAO(), - dao, - USER_PATCH_FIELDS, - USER_UPDATE_FIELDS); + super(UserResource.COLLECTION_PATH, USER, User.class, dao.userDAO(), dao, USER_PATCH_FIELDS, USER_UPDATE_FIELDS); organization = dao.teamDAO().findEntityReferenceByName(Entity.ORGANIZATION_NAME, Include.ALL); } @@ -152,6 +158,20 @@ public class UserRepository extends EntityRepository { return user.withInheritedRoles(fields.contains("roles") ? getInheritedRoles(user) : null); } + @Override + public String exportToCsv(String importingTeam, String user) throws IOException { + Team team = daoCollection.teamDAO().findEntityByName(importingTeam); + return new UserCsv(team, user).exportCsv(); + } + + @Override + public CsvImportResult importFromCsv(String importingTeam, String csv, boolean dryRun, String user) + throws IOException { + Team team = daoCollection.teamDAO().findEntityByName(importingTeam); + UserCsv userCsv = new UserCsv(team, user); + return userCsv.importCsv(csv, dryRun); + } + public boolean isTeamJoinable(String teamId) throws IOException { Team team = daoCollection.teamDAO().findEntityById(UUID.fromString(teamId), Include.NON_DELETED); return team.getIsJoinable(); @@ -204,7 +224,7 @@ public class UserRepository extends EntityRepository { private List getOwns(User user) throws IOException { // Compile entities owned by the user List ownedEntities = - daoCollection.relationshipDAO().findTo(user.getId().toString(), Entity.USER, Relationship.OWNS.ordinal()); + daoCollection.relationshipDAO().findTo(user.getId().toString(), USER, Relationship.OWNS.ordinal()); // Compile entities owned by the team the user belongs to List teams = user.getTeams() == null ? getTeams(user) : user.getTeams(); @@ -218,7 +238,7 @@ public class UserRepository extends EntityRepository { private List getFollows(User user) throws IOException { return EntityUtil.getEntityReferences( - daoCollection.relationshipDAO().findTo(user.getId().toString(), Entity.USER, Relationship.FOLLOWS.ordinal())); + daoCollection.relationshipDAO().findTo(user.getId().toString(), USER, Relationship.FOLLOWS.ordinal())); } private List getTeamChildren(UUID teamId) throws IOException { @@ -252,13 +272,13 @@ public class UserRepository extends EntityRepository { /* Get all the roles that user has been assigned and inherited from the team to User entity */ private List getRoles(User user) throws IOException { - List roleIds = findTo(user.getId(), Entity.USER, Relationship.HAS, Entity.ROLE); + List roleIds = findTo(user.getId(), USER, Relationship.HAS, Entity.ROLE); return EntityUtil.populateEntityReferences(roleIds, Entity.ROLE); } /* Get all the teams that user belongs to User entity */ private List getTeams(User user) throws IOException { - List records = findFrom(user.getId(), Entity.USER, Relationship.HAS, Entity.TEAM); + List records = findFrom(user.getId(), USER, Relationship.HAS, Entity.TEAM); List teams = EntityUtil.populateEntityReferences(records, Entity.TEAM); teams = teams.stream().filter(team -> !team.getDeleted()).collect(Collectors.toList()); // Filter deleted teams // If there are no teams that a user belongs to then return organization as the default team @@ -271,7 +291,7 @@ public class UserRepository extends EntityRepository { private void assignRoles(User user, List roles) { roles = listOrEmpty(roles); for (EntityReference role : roles) { - addRelationship(user.getId(), role.getId(), Entity.USER, Entity.ROLE, Relationship.HAS); + addRelationship(user.getId(), role.getId(), USER, Entity.ROLE, Relationship.HAS); } } @@ -281,7 +301,7 @@ public class UserRepository extends EntityRepository { if (team.getId().equals(organization.getId())) { continue; // Default relationship user to organization team is not stored } - addRelationship(team.getId(), user.getId(), Entity.TEAM, Entity.USER, Relationship.HAS); + addRelationship(team.getId(), user.getId(), Entity.TEAM, USER, Relationship.HAS); } if (teams.size() > 1) { // Remove organization team from the response @@ -290,6 +310,113 @@ public class UserRepository extends EntityRepository { } } + public static class UserCsv extends EntityCsv { + public static final CsvDocumentation DOCUMENTATION = getCsvDocumentation(USER); + public static final List HEADERS = DOCUMENTATION.getHeaders(); + public final Team team; + + UserCsv(Team importingTeam, String updatedBy) { + super(USER, HEADERS, updatedBy); + this.team = importingTeam; + } + + @Override + protected User toEntity(CSVPrinter printer, CSVRecord record) throws IOException { + // Field 1, 2, 3, 4, 5, 6 - name, displayName, description, email, timezone, isAdmin + User user = + new User() + .withName(record.get(0)) + .withDisplayName(record.get(1)) + .withDescription(record.get(2)) + .withEmail(record.get(3)) + .withTimezone(record.get(4)) + .withIsAdmin(getBoolean(printer, record, 5)); + + // Field 7 - team + user.setTeams(getTeams(printer, record, user.getName())); + if (!processRecord) { + return null; + } + + // Field 8 - roles + user.setRoles(getEntityReferences(printer, record, 7, ROLE)); + if (!processRecord) { + return null; + } + + // TODO authentication mechanism? + return user; + } + + @Override + protected List toRecord(User entity) { + // Headers - name,displayName,description,email,timezone,isAdmin,team,roles + List record = new ArrayList<>(); + addField(record, entity.getName()); + addField(record, entity.getDisplayName()); + addField(record, entity.getDescription()); + addField(record, entity.getEmail()); + addField(record, entity.getTimezone()); + addField(record, entity.getIsAdmin()); + addField(record, entity.getTeams().get(0).getFullyQualifiedName()); + addEntityReferences(record, entity.getRoles()); + return record; + } + + private List listUsers( + TeamRepository teamRepository, + UserRepository userRepository, + String parentTeam, + List users, + Fields fields) + throws IOException { + // Export the users by listing users for the entire team hierarchy + ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("team", parentTeam); + + // Add users for the given team + List userList = userRepository.listAll(fields, filter); + if (!nullOrEmpty(userList)) { + users.addAll(userList); + } + + filter = new ListFilter(Include.NON_DELETED).addQueryParam("parentTeam", parentTeam); + List teamList = teamRepository.listAll(Fields.EMPTY_FIELDS, filter); + for (Team team : teamList) { + listUsers(teamRepository, userRepository, team.getName(), users, fields); + } + return users; + } + + public String exportCsv() throws IOException { + UserRepository userRepository = (UserRepository) Entity.getEntityRepository(USER); + TeamRepository teamRepository = (TeamRepository) Entity.getEntityRepository(TEAM); + final Fields fields = userRepository.getFields("roles,teams"); + return exportCsv(listUsers(teamRepository, userRepository, team.getName(), new ArrayList<>(), fields)); + } + + private List getTeams(CSVPrinter printer, CSVRecord record, String user) throws IOException { + List teams = getEntityReferences(printer, record, 6, Entity.TEAM); + + // Validate team being created is under the hierarchy of the team for which CSV is being imported to + for (EntityReference teamRef : listOrEmpty(teams)) { + if (teamRef.getName().equals(team.getName())) { + continue; // Team is same as the team to which CSV is being imported, then it is in the same hierarchy + } + // Else the parent should already exist + if (!SubjectCache.getInstance().isInTeam(team.getName(), listOf(teamRef))) { + importFailure(printer, invalidTeam(6, team.getName(), user, teamRef.getName()), record); + processRecord = false; + } + } + return teams; + } + + public static String invalidTeam(int field, String team, String user, String userTeam) { + String error = String.format("Team %s of user %s is not under %s team hierarchy", userTeam, user, team); + return String.format("#%s: Field %d error - %s", CsvErrorType.INVALID_FIELD, field + 1, error); + } + } + /** Handles entity updated from PUT and POST operation. */ public class UserUpdater extends EntityUpdater { public UserUpdater(User original, User updated, Operation operation) { @@ -311,7 +438,7 @@ public class UserRepository extends EntityRepository { private void updateRoles(User original, User updated) throws IOException { // Remove roles from original and add roles from updated - deleteFrom(original.getId(), Entity.USER, Relationship.HAS, Entity.ROLE); + deleteFrom(original.getId(), USER, Relationship.HAS, Entity.ROLE); assignRoles(updated, updated.getRoles()); List origRoles = listOrEmpty(original.getRoles()); @@ -327,7 +454,7 @@ public class UserRepository extends EntityRepository { private void updateTeams(User original, User updated) throws IOException { // Remove teams from original and add teams from updated - deleteTo(original.getId(), Entity.USER, Relationship.HAS, Entity.TEAM); + deleteTo(original.getId(), USER, Relationship.HAS, Entity.TEAM); assignTeams(updated, updated.getTeams()); List origTeams = listOrEmpty(original.getTeams()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/alerts/AlertResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/alerts/AlertResource.java index 7a195a50e9e..e01d886ea5c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/alerts/AlertResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/alerts/AlertResource.java @@ -66,9 +66,9 @@ import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.alerts.ActivityFeedAlertCache; import org.openmetadata.service.alerts.AlertUtil; import org.openmetadata.service.alerts.AlertsPublisherManager; +import org.openmetadata.service.jdbi3.AlertActionRepository; import org.openmetadata.service.jdbi3.AlertRepository; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; @@ -122,9 +122,10 @@ public class AlertResource extends EntityResource { activityFeedAlert = JsonUtils.readObjects(alertJson, Alert.class).get(0); activityFeedAlert.setId(UUID.randomUUID()); // populate alert actions - EntityRepository actionEntityRepository = Entity.getEntityRepository(Entity.ALERT_ACTION); + AlertActionRepository alertActionRepository = + (AlertActionRepository) Entity.getEntityRepository(Entity.ALERT_ACTION); AlertAction action = - actionEntityRepository.getByName(null, alertActions.getName(), actionEntityRepository.getFields("id")); + alertActionRepository.getByName(null, alertActions.getName(), alertActionRepository.getFields("id")); activityFeedAlert.setAlertActions(List.of(action.getEntityReference())); dao.initializeEntity(activityFeedAlert); } catch (Exception e) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/permissions/PermissionsResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/permissions/PermissionsResource.java index 0a34c6a693e..00e9a648b4f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/permissions/PermissionsResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/permissions/PermissionsResource.java @@ -141,7 +141,7 @@ public class PermissionsResource { @Parameter(description = "Resource type", schema = @Schema(type = "String")) @PathParam("resource") String resource, @Parameter(description = "Entity Id", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) { - EntityRepository entityRepository = Entity.getEntityRepository(resource); + EntityRepository entityRepository = Entity.getEntityRepository(resource); ResourceContext resourceContext = ResourceContext.builder().resource(resource).id(id).entityRepository(entityRepository).build(); return authorizer.getPermission(securityContext, user, resourceContext); @@ -174,7 +174,7 @@ public class PermissionsResource { @Parameter(description = "Resource type", schema = @Schema(type = "String")) @PathParam("resource") String resource, @Parameter(description = "Entity Name", schema = @Schema(type = "String")) @PathParam("name") String name) { - EntityRepository entityRepository = Entity.getEntityRepository(resource); + EntityRepository entityRepository = Entity.getEntityRepository(resource); ResourceContext resourceContext = ResourceContext.builder().resource(resource).name(name).entityRepository(entityRepository).build(); return authorizer.getPermission(securityContext, user, resourceContext); @@ -202,7 +202,7 @@ public class PermissionsResource { throws IOException { // User must have read access to policies OperationContext operationContext = new OperationContext(Entity.POLICY, MetadataOperation.VIEW_ALL); - EntityRepository dao = Entity.getEntityRepository(Entity.POLICY); + EntityRepository dao = Entity.getEntityRepository(Entity.POLICY); for (UUID id : ids) { ResourceContext resourceContext = EntityResource.getResourceContext(Entity.POLICY, dao).id(id).build(); authorizer.authorize(securityContext, operationContext, resourceContext); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelCache.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelCache.java index 12e856ab736..0947912cc53 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelCache.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelCache.java @@ -30,7 +30,10 @@ import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TagLabel.TagSource; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.ClassificationRepository; +import org.openmetadata.service.jdbi3.GlossaryRepository; +import org.openmetadata.service.jdbi3.GlossaryTermRepository; +import org.openmetadata.service.jdbi3.TagRepository; import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.FullyQualifiedName; @@ -42,13 +45,13 @@ public class TagLabelCache { private static final TagLabelCache INSTANCE = new TagLabelCache(); private static volatile boolean INITIALIZED = false; - protected static EntityRepository TAG_REPOSITORY; - protected static EntityRepository TAG_CATEGORY_REPOSITORY; + protected static TagRepository TAG_REPOSITORY; + protected static ClassificationRepository TAG_CLASSIFICATION_REPOSITORY; protected static LoadingCache TAG_CACHE; // Tag fqn to Tag protected static LoadingCache TAG_CATEGORY_CACHE; // Classification name to Classification - protected static EntityRepository GLOSSARY_TERM_REPOSITORY; - protected static EntityRepository GLOSSARY_REPOSITORY; + protected static GlossaryTermRepository GLOSSARY_TERM_REPOSITORY; + protected static GlossaryRepository GLOSSARY_REPOSITORY; protected static LoadingCache GLOSSARY_TERM_CACHE; // Glossary term fqn to GlossaryTerm protected static LoadingCache GLOSSARY_CACHE; // Glossary fqn to Glossary @@ -62,8 +65,8 @@ public class TagLabelCache { .build(new ClassificationLoader()); TAG_CACHE = CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(2, TimeUnit.MINUTES).build(new TagLoader()); - TAG_REPOSITORY = Entity.getEntityRepository(Entity.TAG); - TAG_CATEGORY_REPOSITORY = Entity.getEntityRepository(Entity.CLASSIFICATION); + TAG_REPOSITORY = (TagRepository) Entity.getEntityRepository(Entity.TAG); + TAG_CLASSIFICATION_REPOSITORY = (ClassificationRepository) Entity.getEntityRepository(Entity.CLASSIFICATION); GLOSSARY_CACHE = CacheBuilder.newBuilder().maximumSize(25).expireAfterWrite(2, TimeUnit.MINUTES).build(new GlossaryLoader()); @@ -72,8 +75,8 @@ public class TagLabelCache { .maximumSize(100) .expireAfterWrite(2, TimeUnit.MINUTES) .build(new GlossaryTermLoader()); - GLOSSARY_TERM_REPOSITORY = Entity.getEntityRepository(Entity.GLOSSARY_TERM); - GLOSSARY_REPOSITORY = Entity.getEntityRepository(Entity.GLOSSARY); + GLOSSARY_TERM_REPOSITORY = (GlossaryTermRepository) Entity.getEntityRepository(Entity.GLOSSARY_TERM); + GLOSSARY_REPOSITORY = (GlossaryRepository) Entity.getEntityRepository(Entity.GLOSSARY); INITIALIZED = true; } else { LOG.info("Subject cache is already initialized"); @@ -156,7 +159,7 @@ public class TagLabelCache { static class ClassificationLoader extends CacheLoader { @Override public Classification load(@CheckForNull String categoryName) throws IOException { - Classification category = TAG_CATEGORY_REPOSITORY.getByName(null, categoryName, Fields.EMPTY_FIELDS); + Classification category = TAG_CLASSIFICATION_REPOSITORY.getByName(null, categoryName, Fields.EMPTY_FIELDS); LOG.info("Loaded user {}:{}", category.getName(), category.getId()); return category; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java index def466e3df1..3b3297059ff 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java @@ -52,7 +52,6 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; -import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.api.classification.CreateTag; import org.openmetadata.schema.api.classification.LoadTags; import org.openmetadata.schema.api.data.RestoreEntity; @@ -64,6 +63,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; +import org.openmetadata.service.jdbi3.ClassificationRepository; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; @@ -103,7 +103,8 @@ public class TagResource extends EntityResource { if (!(daoCollection.relationshipDAO().findIfAnyRelationExist(CLASSIFICATION, TAG) > 0)) { // We are missing relationship for classification -> tag, and also tag -> tag (parent relationship) // Find tag definitions and load classifications from the json file, if necessary - EntityRepository classificationRepository = Entity.getEntityRepository(CLASSIFICATION); + ClassificationRepository classificationRepository = + (ClassificationRepository) Entity.getEntityRepository(CLASSIFICATION); try { List classificationList = classificationRepository.listAll(classificationRepository.getFields("*"), new ListFilter(Include.ALL)); @@ -154,7 +155,8 @@ public class TagResource extends EntityResource { // TODO: Once we have migrated to the version above 0.13.1, then this can be removed migrateTags(); // Find tag definitions and load classifications from the json file, if necessary - EntityRepository classificationRepository = Entity.getEntityRepository(CLASSIFICATION); + ClassificationRepository classificationRepository = + (ClassificationRepository) Entity.getEntityRepository(CLASSIFICATION); List loadTagsList = EntityRepository.getEntitiesFromSeedData(CLASSIFICATION, ".*json/data/tags/.*\\.json$", LoadTags.class); for (LoadTags loadTags : loadTagsList) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java index acd6a40f52a..982b03d8a10 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java @@ -48,7 +48,6 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; import lombok.extern.slf4j.Slf4j; -import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.api.data.RestoreEntity; import org.openmetadata.schema.api.teams.CreateRole; import org.openmetadata.schema.entity.teams.Role; @@ -58,7 +57,6 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.RoleRepository; import org.openmetadata.service.resources.Collection; @@ -436,7 +434,7 @@ public class RoleResource extends EntityResource { } public static EntityReference getRole(String roleName) { - EntityRepository dao = Entity.getEntityRepository(Entity.ROLE); - return dao.dao.findEntityReferenceByName(roleName); + RoleRepository roleRepository = (RoleRepository) Entity.getEntityRepository(Entity.ROLE); + return roleRepository.dao.findEntityReferenceByName(roleName); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java index a641983b67a..f4f2dbbfe30 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java @@ -457,7 +457,10 @@ public class TeamResource extends EntityResource { @GET @Path("/documentation/csv") @Valid - @Operation(operationId = "getCsvDocumentation", summary = "Get CSV documentation", tags = "glossaries") + @Operation( + operationId = "getCsvDocumentation", + summary = "Get CSV documentation for team import/export", + tags = "teams") public String getCsvDocumentation(@Context SecurityContext securityContext, @PathParam("name") String name) throws IOException { return JsonUtils.pojoToJson(TeamCsv.DOCUMENTATION); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java index 737b5b37333..fe343293aa7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java @@ -93,6 +93,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.ProviderType; import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.csv.CsvImportResult; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.CatalogExceptionMessage; @@ -101,6 +102,7 @@ import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.TokenRepository; import org.openmetadata.service.jdbi3.UserRepository; +import org.openmetadata.service.jdbi3.UserRepository.UserCsv; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.resources.EntityResource; import org.openmetadata.service.secrets.SecretsManager; @@ -1027,6 +1029,79 @@ public class UserResource extends EntityResource { return Response.status(Response.Status.OK).entity(authHandler.getNewAccessToken(refreshRequest)).build(); } + @GET + @Path("/documentation/csv") + @Valid + @Operation( + operationId = "getCsvDocumentation", + summary = "Get CSV documentation for user import/export", + tags = "users") + public String getUserCsvDocumentation(@Context SecurityContext securityContext, @PathParam("name") String name) + throws IOException { + return JsonUtils.pojoToJson(UserCsv.DOCUMENTATION); + } + + @GET + @Path("/export") + @Produces(MediaType.TEXT_PLAIN) + @Valid + @Operation( + operationId = "exportUsers", + summary = "Export users in a team in CSV format", + tags = "users", + responses = { + @ApiResponse( + responseCode = "200", + description = "Exported csv with user information", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = String.class))) + }) + public String exportUsersCsv( + @Context SecurityContext securityContext, + @Parameter( + description = "Name of the team to under which the users are imported to", + required = true, + schema = @Schema(type = "string")) + @QueryParam("team") + String team) + throws IOException { + return exportCsvInternal(securityContext, team); + } + + @PUT + @Path("/import") + @Consumes(MediaType.TEXT_PLAIN) + @Valid + @Operation( + operationId = "importTeams", + summary = "Import from CSV to create, and update teams.", + tags = "users", + responses = { + @ApiResponse( + responseCode = "200", + description = "Import result", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = CsvImportResult.class))) + }) + public CsvImportResult importCsv( + @Context SecurityContext securityContext, + @Parameter( + description = "Name of the team to under which the users are imported to", + required = true, + schema = @Schema(type = "string")) + @QueryParam("team") + String team, + @Parameter( + description = + "Dry-run when true is used for validating the CSV without really importing it. (default=true)", + schema = @Schema(type = "boolean")) + @DefaultValue("true") + @QueryParam("dryRun") + boolean dryRun, + String csv) + throws IOException { + return importCsvInternal(securityContext, team, csv, dryRun); + } + private User getUser(SecurityContext securityContext, CreateUser create) { return new User() .withId(UUID.randomUUID()) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java index 191ae1f4658..9c467321ce7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java @@ -28,9 +28,10 @@ import org.openmetadata.schema.entity.teams.AuthenticationMechanism; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.SecretsManagerUpdateException; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.IngestionPipelineRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.ServiceEntityRepository; +import org.openmetadata.service.jdbi3.UserRepository; import org.openmetadata.service.resources.CollectionRegistry; import org.openmetadata.service.resources.CollectionRegistry.CollectionDetails; import org.openmetadata.service.resources.services.ServiceEntityResource; @@ -49,8 +50,8 @@ import org.openmetadata.service.util.EntityUtil; public class SecretsManagerUpdateService { private final SecretsManager secretManager; private final SecretsManager oldSecretManager; - private final EntityRepository userRepository; - private final EntityRepository ingestionPipelineRepository; + private final UserRepository userRepository; + private final IngestionPipelineRepository ingestionPipelineRepository; private final Map, ServiceEntityRepository> connectionTypeRepositoriesMap; @@ -58,8 +59,9 @@ public class SecretsManagerUpdateService { public SecretsManagerUpdateService(SecretsManager secretsManager, String clusterName) { this.secretManager = secretsManager; this.connectionTypeRepositoriesMap = retrieveConnectionTypeRepositoriesMap(); - this.userRepository = Entity.getEntityRepository(Entity.USER); - this.ingestionPipelineRepository = Entity.getEntityRepository(Entity.INGESTION_PIPELINE); + this.userRepository = (UserRepository) Entity.getEntityRepository(Entity.USER); + this.ingestionPipelineRepository = + (IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE); // by default, it is going to be non-managed secrets manager since decrypt is the same for all of them this.oldSecretManager = SecretsManagerFactory.createSecretsManager(null, clusterName); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java index 23d495544fe..9fd60a365da 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java @@ -25,7 +25,7 @@ import org.openmetadata.schema.type.ResourcePermission; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.UserRepository; import org.openmetadata.service.security.policyevaluator.OperationContext; import org.openmetadata.service.security.policyevaluator.PolicyEvaluator; import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; @@ -84,7 +84,7 @@ public class NoopAuthorizer implements Authorizer { private void addOrUpdateUser(User user) { try { - EntityRepository userRepository = Entity.getEntityRepository(Entity.USER); + UserRepository userRepository = (UserRepository) Entity.getEntityRepository(Entity.USER); RestUtil.PutResponse addedUser = userRepository.createOrUpdate(null, user); LOG.debug("Added anonymous user entry: {}", addedUser); } catch (IOException exception) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/PolicyCache.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/PolicyCache.java index b2864b9d8e2..14fbc88aac0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/PolicyCache.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/PolicyCache.java @@ -28,7 +28,7 @@ import org.openmetadata.schema.entity.policies.Policy; import org.openmetadata.schema.entity.policies.accessControl.Rule; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.PolicyRepository; import org.openmetadata.service.util.EntityUtil.Fields; /** Subject context used for Access Control Policies */ @@ -38,7 +38,7 @@ public class PolicyCache { private static volatile boolean INITIALIZED = false; protected static LoadingCache> POLICY_CACHE; - private static EntityRepository POLICY_REPOSITORY; + private static PolicyRepository POLICY_REPOSITORY; private static Fields FIELDS; public static PolicyCache getInstance() { @@ -49,7 +49,7 @@ public class PolicyCache { public static void initialize() { if (!INITIALIZED) { POLICY_CACHE = CacheBuilder.newBuilder().maximumSize(100).build(new PolicyLoader()); - POLICY_REPOSITORY = Entity.getEntityRepository(Entity.POLICY); + POLICY_REPOSITORY = (PolicyRepository) Entity.getEntityRepository(Entity.POLICY); FIELDS = POLICY_REPOSITORY.getFields("rules"); INITIALIZED = true; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/RoleCache.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/RoleCache.java index 36ff921b44b..7ebe6683264 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/RoleCache.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/RoleCache.java @@ -25,7 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.entity.teams.Role; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.RoleRepository; import org.openmetadata.service.util.EntityUtil.Fields; /** Subject context used for Access Control Policies */ @@ -34,7 +34,7 @@ public class RoleCache { private static final RoleCache INSTANCE = new RoleCache(); private static volatile boolean INITIALIZED = false; protected static LoadingCache ROLE_CACHE; - private static EntityRepository ROLE_REPOSITORY; + private static RoleRepository ROLE_REPOSITORY; private static Fields FIELDS; public static RoleCache getInstance() { @@ -45,7 +45,7 @@ public class RoleCache { public static void initialize() { if (!INITIALIZED) { ROLE_CACHE = CacheBuilder.newBuilder().maximumSize(100).build(new RoleLoader()); - ROLE_REPOSITORY = Entity.getEntityRepository(Entity.ROLE); + ROLE_REPOSITORY = (RoleRepository) Entity.getEntityRepository(Entity.ROLE); FIELDS = ROLE_REPOSITORY.getFields("policies"); INITIALIZED = true; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/SubjectCache.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/SubjectCache.java index 9f66bfa0ccd..ac523c13891 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/SubjectCache.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/SubjectCache.java @@ -35,7 +35,8 @@ import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.TeamRepository; +import org.openmetadata.service.jdbi3.UserRepository; import org.openmetadata.service.util.EntityUtil.Fields; /** Subject context used for Access Control Policies */ @@ -46,9 +47,9 @@ public class SubjectCache { protected static LoadingCache USER_CACHE; protected static LoadingCache USER_CACHE_WIH_ID; protected static LoadingCache TEAM_CACHE; - protected static EntityRepository USER_REPOSITORY; + protected static UserRepository USER_REPOSITORY; protected static Fields USER_FIELDS; - protected static EntityRepository TEAM_REPOSITORY; + protected static TeamRepository TEAM_REPOSITORY; protected static Fields TEAM_FIELDS; // Expected to be called only once from the DefaultAuthorizer @@ -63,9 +64,9 @@ public class SubjectCache { .build(new UserLoaderWithId()); TEAM_CACHE = CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new TeamLoader()); - USER_REPOSITORY = Entity.getEntityRepository(Entity.USER); + USER_REPOSITORY = (UserRepository) Entity.getEntityRepository(Entity.USER); USER_FIELDS = USER_REPOSITORY.getFields("roles, teams, isAdmin"); - TEAM_REPOSITORY = Entity.getEntityRepository(Entity.TEAM); + TEAM_REPOSITORY = (TeamRepository) Entity.getEntityRepository(Entity.TEAM); TEAM_FIELDS = TEAM_REPOSITORY.getFields("defaultRoles, policies, parents"); INSTANCE = new SubjectCache(); INITIALIZED = true; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/TestCaseResourceContext.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/TestCaseResourceContext.java index 9aa05519b64..ecbc6f0462e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/TestCaseResourceContext.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/TestCaseResourceContext.java @@ -26,6 +26,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.TestCaseRepository; import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; import org.openmetadata.service.util.EntityUtil; @@ -91,14 +92,14 @@ public class TestCaseResourceContext implements ResourceContextInterface { } private static EntityInterface resolveEntityById(UUID id) throws IOException { - EntityRepository dao = Entity.getEntityRepository(Entity.TEST_CASE); + TestCaseRepository dao = (TestCaseRepository) Entity.getEntityRepository(Entity.TEST_CASE); TestCase testCase = dao.get(null, id, dao.getFields("entityLink"), Include.ALL); return resolveEntityByEntityLink(EntityLink.parse(testCase.getEntityLink())); } private static EntityInterface resolveEntityByName(String fqn) throws IOException { if (fqn == null) return null; - EntityRepository dao = Entity.getEntityRepository(Entity.TEST_CASE); + TestCaseRepository dao = (TestCaseRepository) Entity.getEntityRepository(Entity.TEST_CASE); TestCase testCase = dao.getByName(null, fqn, dao.getFields("entityLink"), Include.ALL); return resolveEntityByEntityLink(EntityLink.parse(testCase.getEntityLink())); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/ElasticSearchIndexUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/ElasticSearchIndexUtil.java index 0a899cf5368..63634bbef11 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/ElasticSearchIndexUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/ElasticSearchIndexUtil.java @@ -257,10 +257,10 @@ public class ElasticSearchIndexUtil { indexType); } else { // Start fetching a list of Entities and pushing them to ES - EntityRepository entityRepository = Entity.getEntityRepository(entityType); + EntityRepository entityRepository = Entity.getEntityRepository(entityType); List allowedFields = entityRepository.getAllowedFields(); String fields = String.join(",", allowedFields); - ResultList result; + ResultList result; String after = null; try { do { @@ -309,7 +309,7 @@ public class ElasticSearchIndexUtil { ElasticSearchIndexDefinition.ElasticSearchIndexType indexType, BulkProcessor bulkProcessor, String entityType, - List entities) { + List entities) { for (EntityInterface entity : entities) { if (entityType.equals(TABLE)) { ((Table) entity).getColumns().forEach(table -> table.setProfile(null)); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/NotificationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/NotificationHandler.java index 241e64defb5..9357f8e24b0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/NotificationHandler.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/NotificationHandler.java @@ -39,7 +39,7 @@ import org.openmetadata.schema.type.Post; import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.UserRepository; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.resources.settings.SettingsCache; import org.openmetadata.service.socket.WebSocketManager; @@ -160,7 +160,7 @@ public class NotificationHandler { } private void handleEmailNotifications(HashSet userList, Thread thread) { - EntityRepository repository = Entity.getEntityRepository(USER); + UserRepository repository = (UserRepository) Entity.getEntityRepository(USER); URI urlInstance = thread.getHref(); userList.forEach( (id) -> { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java index 6ad123b043d..1df93488ea6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java @@ -62,7 +62,7 @@ public final class UserUtil { } public static void addUserForBasicAuth(String username, String pwd, String domain) throws IOException { - EntityRepository userRepository = Entity.getEntityRepository(Entity.USER); + UserRepository userRepository = (UserRepository) Entity.getEntityRepository(Entity.USER); User originalUser; try { List fields = userRepository.getAllowedFieldsCopy(); @@ -97,7 +97,7 @@ public final class UserUtil { } public static User addOrUpdateUser(User user) { - EntityRepository userRepository = Entity.getEntityRepository(Entity.USER); + UserRepository userRepository = (UserRepository) Entity.getEntityRepository(Entity.USER); try { RestUtil.PutResponse addedUser = userRepository.createOrUpdate(null, user); // should not log the user auth details in LOGS diff --git a/openmetadata-service/src/main/resources/json/data/user/userCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/user/userCsvDocumentation.json new file mode 100644 index 00000000000..3a0932bd10e --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/user/userCsvDocumentation.json @@ -0,0 +1,76 @@ +{ + "summary": "Documentation for CSV file used for importing and exporting users under a team. Users can be imported to any team that is children of the team for which CSV is imported for.", + "headers": [ + { + "name": "name", + "required": true, + "description": "The name of the user being created. This is same as the login name.", + "examples": [ + "`john`", + "`adam.smith`" + ] + }, + { + "name": "displayName", + "required": false, + "description": "Display name for the user.", + "examples": [ + "`John`", + "`Adam Smith`" + ] + }, + { + "name": "description", + "required": false, + "description": "Description for the user in markdown format.", + "examples": [ + "`John` is a Data Scientist." + ] + }, + { + "name": "email", + "required": true, + "description": "Email address of the user.", + "examples": [ + "`john@company.com`" + ] + }, + { + "name": "timezone", + "required": false, + "description": "The timezone of the user.", + "examples": [ + "`America/Los_Angeles`", + "`Brazil/East`" + ] + }, + { + "name": "isAdmin", + "required": false, + "description": "Set true if the user is an Admin. Default - false", + "examples": [ + "`true` or `false`", + "\"\" will use default value `false`" + ] + }, + { + "name": "teams", + "required": true, + "description": "Teams the user belongs to. For multiple teams, provide team names separated by ';'", + "examples": [ + "Marketing team", + "Marketing team;Finance team" + ] + }, + { + "name": "Roles", + "required": false, + "description": "Roles that are assigned to a user.", + "examples": [ + "`Data consumer`", + "`Data consumer;Data steward`", + "`\"\" for no role`" + ] + } + ] +} \ No newline at end of file diff --git a/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java b/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java index 0ee3d36a9ca..c0ec4b18e26 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java @@ -74,7 +74,8 @@ public class CsvUtilTest { // Break a csv text into records, sort it and compare List expectedCsvRecords = listOf(expectedCsv.split(CsvUtil.LINE_SEPARATOR)); List actualCsvRecords = listOf(actualCsv.split(CsvUtil.LINE_SEPARATOR)); - assertEquals(expectedCsvRecords.size(), actualCsvRecords.size()); + assertEquals( + expectedCsvRecords.size(), actualCsvRecords.size(), "Expected " + expectedCsv + " actual " + actualCsv); Collections.sort(expectedCsvRecords); Collections.sort(actualCsvRecords); for (int i = 0; i < expectedCsvRecords.size(); i++) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java b/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java index 8f82c9ed965..0741d5424df 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java @@ -90,7 +90,7 @@ public class EntityCsvTest { int expectedRowsProcessed, int expectedRowsPassed, int expectedRowsFailed) { - assertEquals(expectedStatus, importResult.getStatus(), importResult.getImportResultsCsv()); + assertEquals(expectedStatus, importResult.getStatus(), importResult.toString()); assertEquals(expectedRowsProcessed, importResult.getNumberOfRowsProcessed()); assertEquals(expectedRowsPassed, importResult.getNumberOfRowsPassed()); assertEquals(expectedRowsFailed, importResult.getNumberOfRowsFailed()); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index ac2a5f5c139..96fc7620cfe 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -2027,7 +2027,7 @@ public abstract class EntityResourceTest eventHolder.hasExpectedEvent( @@ -2379,7 +2379,7 @@ public abstract class EntityResourceTest { String record3 = getRecord(3, GROUP, team.getName(), null, true, null, (List) null); List newRecords = listOf(record3); testImportExport(team.getName(), TeamCsv.HEADERS, createRecords, updateRecords, newRecords); + + // Import to team111 a user with parent team1 - since team1 is not under team111 hierarchy, import should fail + String record4 = getRecord(3, GROUP, "x1", null, true, null, (List) null); + String csv = EntityCsvTest.createCsv(TeamCsv.HEADERS, listOf(record4), null); + CsvImportResult result = importCsv("x111", csv, false); + String error = TeamCsv.invalidTeam(4, "x111", "x3", "x1"); + assertTrue(result.getImportResultsCsv().contains(error)); } private static void validateTeam( diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java index f29b8d4ec7f..042053e2412 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java @@ -30,6 +30,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.csv.CsvUtil.recordToString; +import static org.openmetadata.csv.EntityCsvTest.assertRows; +import static org.openmetadata.csv.EntityCsvTest.assertSummary; +import static org.openmetadata.csv.EntityCsvTest.createCsv; +import static org.openmetadata.csv.EntityCsvTest.getFailedRecord; import static org.openmetadata.service.exception.CatalogExceptionMessage.PASSWORD_INVALID_FORMAT; import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound; import static org.openmetadata.service.exception.CatalogExceptionMessage.notAdmin; @@ -71,6 +76,8 @@ import java.util.TimeZone; import java.util.UUID; import java.util.function.Predicate; import java.util.stream.Collectors; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response.Status; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.http.client.HttpResponseException; @@ -78,6 +85,8 @@ import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.TestMethodOrder; +import org.openmetadata.csv.EntityCsv; +import org.openmetadata.csv.EntityCsvTest; import org.openmetadata.schema.api.CreateBot; import org.openmetadata.schema.api.teams.CreateUser; import org.openmetadata.schema.auth.GenerateTokenRequest; @@ -99,9 +108,12 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.ImageList; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.Profile; +import org.openmetadata.schema.type.csv.CsvImportResult; import org.openmetadata.service.Entity; import org.openmetadata.service.auth.JwtResponse; import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.jdbi3.TeamRepository.TeamCsv; +import org.openmetadata.service.jdbi3.UserRepository.UserCsv; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.bots.BotResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; @@ -466,10 +478,6 @@ public class UserResourceTest extends EntityResourceTest { assertEquals(user1, users.getData().get(0)); } - private CreateUser createBotUserRequest(TestInfo test, int index) { - return createBotUserRequest(getEntityName(test, index)); - } - @Test void get_listUsersWithTeamsPagination(TestInfo test) throws IOException { TeamResourceTest teamResourceTest = new TeamResourceTest(); @@ -532,6 +540,12 @@ public class UserResourceTest extends EntityResourceTest { assertEquals(user1, users.getData().get(0)); } + @Test + void get_generateRandomPassword() throws HttpResponseException { + String randomPwd = TestUtils.get(getResource("users/generateRandomPwd"), String.class, ADMIN_AUTH_HEADERS); + assertDoesNotThrow(() -> PasswordUtil.validatePassword(randomPwd), PASSWORD_INVALID_FORMAT); + } + /** * @see EntityResourceTest put_addDeleteFollower_200 test for tests related to GET user with owns field parameter * @see EntityResourceTest put_addDeleteFollower_200 for tests related getting user with follows list @@ -770,12 +784,6 @@ public class UserResourceTest extends EntityResourceTest { assertEquals(StringUtils.EMPTY, jwtAuthMechanism.getJWTToken()); } - @Test - void get_generateRandomPassword() throws HttpResponseException { - String randomPwd = TestUtils.get(getResource("users/generateRandomPwd"), String.class, ADMIN_AUTH_HEADERS); - assertDoesNotThrow(() -> PasswordUtil.validatePassword(randomPwd), PASSWORD_INVALID_FORMAT); - } - @Test void post_createUser_BasicAuth_AdminCreate_login_200_ok(TestInfo test) throws HttpResponseException { // Create a user with Auth and Try Logging in @@ -887,6 +895,74 @@ public class UserResourceTest extends EntityResourceTest { CatalogExceptionMessage.INVALID_USERNAME_PASSWORD); } + @Test + void testCsvDocumentation() throws HttpResponseException { + assertEquals(UserCsv.DOCUMENTATION, getCsvDocumentation()); + } + + @Test + void testImportInvalidCsv() throws IOException { + // Headers - name,displayName,description,email,timezone,isAdmin,teams,roles + TeamResourceTest teamResourceTest = new TeamResourceTest(); + Team team = teamResourceTest.createEntity(teamResourceTest.createRequest("team-invalidCsv"), ADMIN_AUTH_HEADERS); + + // Invalid team + String resultsHeader = recordToString(EntityCsv.getResultHeaders(UserCsv.HEADERS)); + String record = "user,,,user@domain.com,,,invalidTeam,"; + String csv = createCsv(UserCsv.HEADERS, listOf(record), null); + CsvImportResult result = importCsv(team.getName(), csv, false); + assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + String[] expectedRows = {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(6, "invalidTeam"))}; + assertRows(result, expectedRows); + + // Invalid roles + record = "user,,,user@domain.com,,,team-invalidCsv,invalidRole"; + csv = createCsv(UserCsv.HEADERS, listOf(record), null); + result = importCsv(team.getName(), csv, false); + assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + expectedRows = new String[] {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(7, "invalidRole"))}; + assertRows(result, expectedRows); + } + + @Test + void testUserImportExport() throws IOException { + // Create team hierarchy - team with children t1, t1 has t11 + // "name", "displayName", "description", "teamType", "parents", "owner", "isJoinable", "defaultRoles", & "policies" + TeamResourceTest teamResourceTest = new TeamResourceTest(); + String team = "teamImportExport,,,Division,Organization,,,,"; + String team1 = "teamImportExport1,,,Department,teamImportExport,,,,"; + String team11 = "teamImportExport11,,,Group,teamImportExport1,,,,"; + String csv = EntityCsvTest.createCsv(TeamCsv.HEADERS, listOf(team, team1, team11), null); + CsvImportResult result = teamResourceTest.importCsv(ORG_TEAM.getName(), csv, false); + assertEquals(0, result.getNumberOfRowsFailed()); + + // Create users in the team hierarchy + // Headers - name,displayName,description,email,timezone,isAdmin,teams,roles + String user = "userImportExport,d,s,userImportExport@domain.com,America/Los_Angeles,true,teamImportExport,"; + String user1 = "userImportExport1,,,userImportExport1@domain.com,,,teamImportExport1,DataConsumer"; + String user11 = "userImportExport11,,,userImportExport11@domain.com,,,teamImportExport11,"; + List createRecords = listOf(user, user1, user11); + + // Update user descriptions + user = "userImportExport,displayName,,userImportExport@domain.com,,,teamImportExport,"; + user1 = "userImportExport1,displayName1,,userImportExport1@domain.com,,,teamImportExport1,"; + user11 = "userImportExport11,displayName11,,userImportExport11@domain.com,,,teamImportExport11,"; + List updateRecords = listOf(user, user1, user11); + + // Add new users + String user2 = "userImportExport2,displayName2,,userImportExport2@domain.com,,,teamImportExport1,"; + String user21 = "userImportExport21,displayName21,,userImportExport11@domain.com,,,teamImportExport11,"; + List newRecords = listOf(user2, user21); + testImportExport("teamImportExport", UserCsv.HEADERS, createRecords, updateRecords, newRecords); + + // Import to team11 a user in team1 - since team1 is not under team11 hierarchy, import should fail + String user3 = "userImportExport3,displayName3,,userImportExport3@domain.com,,,teamImportExport1,"; + csv = EntityCsvTest.createCsv(UserCsv.HEADERS, listOf(user3), null); + result = importCsv("teamImportExport11", csv, false); + String error = UserCsv.invalidTeam(6, "teamImportExport11", "userImportExport3", "teamImportExport1"); + assertTrue(result.getImportResultsCsv().contains(error)); + } + private String encodePassword(String password) { return Base64.getEncoder().encodeToString(password.getBytes()); } @@ -1113,4 +1189,23 @@ public class UserResourceTest extends EntityResourceTest { .withAuthType(AuthenticationMechanism.AuthType.JWT) .withConfig(new JWTAuthMechanism().withJWTTokenExpiry(JWTTokenExpiry.Unlimited))); } + + private CreateUser createBotUserRequest(TestInfo test, int index) { + return createBotUserRequest(getEntityName(test, index)); + } + + @Override + public CsvImportResult importCsv(String teamName, String csv, boolean dryRun) throws HttpResponseException { + WebTarget target = getCollection().path("/import"); + target = target.queryParam("team", teamName); + target = !dryRun ? target.queryParam("dryRun", false) : target; + return TestUtils.putCsv(target, csv, CsvImportResult.class, Status.OK, ADMIN_AUTH_HEADERS); + } + + @Override + protected String exportCsv(String teamName) throws HttpResponseException { + WebTarget target = getCollection().path("/export"); + target = target.queryParam("team", teamName); + return TestUtils.get(target, String.class, ADMIN_AUTH_HEADERS); + } } diff --git a/openmetadata-spec/src/main/resources/json/schema/type/basic.json b/openmetadata-spec/src/main/resources/json/schema/type/basic.json index bdbe6733987..410b7255bbb 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/basic.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/basic.json @@ -85,6 +85,11 @@ "type": "string", "format": "time" }, + "timezone": { + "description": "Timezone of the user in the format `America/Los_Angeles`, `Brazil/East`, etc.", + "type": "string", + "format": "timezone" + }, "entityLink": { "description": "Link to an entity or field within an entity using this format `<#E::{entities}::{entityType}::{field}::{arrayFieldName}::{arrayFieldValue}`.", "type": "string",