yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-04 11:33:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes 10021 - Add CSV import/export for users (#10022)

Browse Source 
* Fixes 10021 - Add CSV import/export for users

* Fixes 10021 - Add CSV import/export for users
This commit is contained in:
Suresh Srinivas 2023-01-30 21:34:34 -08:00 committed by GitHub
parent 28c8ce1386
commit a1b9d3fe65
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 509 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
common/src/main/java/org/openmetadata/common/utils/CommonUtil.java
View File

@ -24,6 +24,7 @@ import java.nio.file.Paths;
import java.text.DateFormat;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Calendar;
import java.util.Collection;
@ -170,12 +171,12 @@ public final class CommonUtil {
return IOUtils.toString(Objects.requireNonNull(loader.getResourceAsStream(file)), UTF_8);
}
/** Return list of entiries that are modifiable for performing sort and other operations */
/** Return list of entries that are modifiable for performing sort and other operations */
@SafeVarargs
public static <T> List<T> listOf(T... entries) {
if (entries == null) {
return Collections.emptyList();
}
return new ArrayList<>(List.of(entries));
return new ArrayList<>(Arrays.asList(entries));
}
}

4
openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java
View File

@ -97,9 +97,7 @@ public final class CsvUtil {
}
public static List<String> addField(List<String> record, Boolean field) {
if (field != null) {
record.add(field.toString());
}
record.add(field == null ? "" : field.toString());
return record;
}

4
openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java
View File

@ -305,12 +305,12 @@ public abstract class EntityCsv<T extends EntityInterface> {
entity.setId(UUID.randomUUID());
entity.setUpdatedBy(importedBy);
entity.setUpdatedAt(System.currentTimeMillis());
EntityRepository<EntityInterface> repository = Entity.getEntityRepository(entityType);
EntityRepository<T> repository = (EntityRepository<T>) Entity.getEntityRepository(entityType);
Response.Status responseStatus;
if (!importResult.getDryRun()) {
try {
repository.prepareInternal(entity);
PutResponse<EntityInterface> response = repository.createOrUpdate(null, entity);
PutResponse<T> response = repository.createOrUpdate(null, entity);
responseStatus = response.getStatus();
} catch (Exception ex) {
importFailure(resultsPrinter, ex.getMessage(), record);

12
openmetadata-service/src/main/java/org/openmetadata/service/Entity.java
View File

@ -48,7 +48,7 @@ public final class Entity {
private static final Map<String, EntityDAO<?>> DAO_MAP = new HashMap<>();
// Canonical entity name to corresponding EntityRepository map
private static final Map<String, EntityRepository<?>> ENTITY_REPOSITORY_MAP = new HashMap<>();
private static final Map<String, EntityRepository<? extends EntityInterface>> ENTITY_REPOSITORY_MAP = new HashMap<>();
// List of all the entities
private static final List<String> ENTITY_LIST = new ArrayList<>();
@ -258,10 +258,14 @@ public final class Entity {
return entity;
}
/** Retrieve the corresponding entity repository for a given entity name. */
public static <T extends EntityInterface> EntityRepository<T> getEntityRepository(@NonNull String entityType) {
/**
* Retrieve the corresponding entity repository for a given entity name.
*
* @return
*/
public static EntityRepository<? extends EntityInterface> getEntityRepository(@NonNull String entityType) {
@SuppressWarnings("unchecked")
EntityRepository<T> entityRepository = (EntityRepository<T>) ENTITY_REPOSITORY_MAP.get(entityType);
EntityRepository<? extends EntityInterface> entityRepository = ENTITY_REPOSITORY_MAP.get(entityType);
if (entityRepository == null) {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityTypeNotFound(entityType));
}

6
openmetadata-service/src/main/java/org/openmetadata/service/alerts/AlertsPublisherManager.java
View File

@ -18,8 +18,8 @@ import org.openmetadata.schema.entity.alerts.AlertActionStatus;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.Entity;
import org.openmetadata.service.events.EventPubSub;
import org.openmetadata.service.jdbi3.AlertActionRepository;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.EntityRepository;
@Slf4j
public class AlertsPublisherManager {
@ -47,10 +47,10 @@ public class AlertsPublisherManager {
}
public void addAlertActionPublishers(Alert alert) throws IOException {
EntityRepository<AlertAction> alertActionEntityRepository = Entity.getEntityRepository(ALERT_ACTION);
AlertActionRepository alertActionRepository = (AlertActionRepository) Entity.getEntityRepository(ALERT_ACTION);
for (EntityReference alertActionRef : alert.getAlertActions()) {
AlertAction action =
alertActionEntityRepository.get(null, alertActionRef.getId(), alertActionEntityRepository.getFields("*"));
alertActionRepository.get(null, alertActionRef.getId(), alertActionRepository.getFields("*"));
addAlertActionPublisher(alert, action);
}
}

4
openmetadata-service/src/main/java/org/openmetadata/service/alerts/emailAlert/EmailAlertPublisher.java
View File

@ -20,8 +20,8 @@ import org.openmetadata.service.Entity;
import org.openmetadata.service.alerts.AlertsActionPublisher;
import org.openmetadata.service.events.errors.EventPublisherException;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.UserRepository;
import org.openmetadata.service.resources.events.EventResource;
import org.openmetadata.service.security.policyevaluator.SubjectCache;
import org.openmetadata.service.util.ChangeEventParser;
@ -74,7 +74,7 @@ public class EmailAlertPublisher extends AlertsActionPublisher {
private Set<String> sendToAdmins() {
Set<String> emailList = new HashSet<>();
EntityRepository<User> userEntityRepository = Entity.getEntityRepository(USER);
UserRepository userEntityRepository = (UserRepository) Entity.getEntityRepository(USER);
ResultList<User> result;
ListFilter listFilter = new ListFilter(Include.ALL);
listFilter.addQueryParam("isAdmin", "true");

2
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AlertRepository.java
View File

@ -102,7 +102,7 @@ public class AlertRepository extends EntityRepository<Alert> {
List<AlertAction> alertActionList = new ArrayList<>();
List<CollectionDAO.EntityRelationshipRecord> records =
daoCollection.relationshipDAO().findTo(alertId.toString(), ALERT, CONTAINS.ordinal(), ALERT_ACTION);
EntityRepository<AlertAction> alertEntityRepository = Entity.getEntityRepository(ALERT_ACTION);
AlertActionRepository alertEntityRepository = (AlertActionRepository) Entity.getEntityRepository(ALERT_ACTION);
for (CollectionDAO.EntityRelationshipRecord record : records) {
AlertAction alertAction = alertEntityRepository.get(null, record.getId(), alertEntityRepository.getFields("*"));
alertAction.setStatusDetails(getActionStatus(alertId, alertAction.getId()));

3
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java
View File

@ -124,10 +124,11 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
return new GlossaryUpdater(original, updated, operation);
}
/** Export glossary as CSV */
@Override
public String exportToCsv(String name, String user) throws IOException {
Glossary glossary = getByName(null, name, Fields.EMPTY_FIELDS); // Validate glossary name
EntityRepository<GlossaryTerm> repository = Entity.getEntityRepository(Entity.GLOSSARY_TERM);
GlossaryTermRepository repository = (GlossaryTermRepository) Entity.getEntityRepository(Entity.GLOSSARY_TERM);
ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parent", name);
List<GlossaryTerm> terms = repository.listAll(repository.getFields("reviewers,tags,relatedTerms"), filter);
terms.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName));

3
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/KpiRepository.java
View File

@ -55,7 +55,8 @@ public class KpiRepository extends EntityRepository<Kpi> {
public void prepare(Kpi kpi) throws IOException {
// validate targetDefinition
Entity.getEntityReferenceById(Entity.DATA_INSIGHT_CHART, kpi.getDataInsightChart().getId(), Include.NON_DELETED);
EntityRepository<DataInsightChart> dataInsightChartRepository = Entity.getEntityRepository(DATA_INSIGHT_CHART);
DataInsightChartRepository dataInsightChartRepository =
(DataInsightChartRepository) Entity.getEntityRepository(DATA_INSIGHT_CHART);
DataInsightChart chart =
dataInsightChartRepository.get(
null, kpi.getDataInsightChart().getId(), dataInsightChartRepository.getFields("metrics"));

13
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java
View File

@ -194,17 +194,15 @@ public class TeamRepository extends EntityRepository<Team> {
SubjectCache.getInstance().invalidateTeam(team.getId());
}
/** Export team as CSV */
@Override
public String exportToCsv(String parentTeam, String user) throws IOException {
Team team = getByName(null, parentTeam, Fields.EMPTY_FIELDS); // Validate glossary name
return new TeamCsv(team, user).exportCsv(this);
Team team = getByName(null, parentTeam, Fields.EMPTY_FIELDS); // Validate team name
return new TeamCsv(team, user).exportCsv();
}
/** Load CSV provided for bulk upload */
@Override
public CsvImportResult importFromCsv(String name, String csv, boolean dryRun, String user) throws IOException {
Team team = getByName(null, name, Fields.EMPTY_FIELDS); // Validate glossary name
Team team = getByName(null, name, Fields.EMPTY_FIELDS); // Validate team name
TeamCsv teamCsv = new TeamCsv(team, user);
return teamCsv.importCsv(csv, dryRun);
}
@ -610,7 +608,7 @@ public class TeamRepository extends EntityRepository<Team> {
throws IOException {
// Export the entire hierarchy of teams
final ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parentTeam", parentTeam);
List<Team> list = repository.listAfter(null, fields, filter, 10000, null).getData();
List<Team> list = repository.listAll(fields, filter);
if (nullOrEmpty(list)) {
return teams;
}
@ -621,7 +619,8 @@ public class TeamRepository extends EntityRepository<Team> {
return teams;
}
public String exportCsv(TeamRepository repository) throws IOException {
public String exportCsv() throws IOException {
TeamRepository repository = (TeamRepository) Entity.getEntityRepository(TEAM);
final Fields fields = repository.getFields("owner,defaultRoles,parents,policies");
return exportCsv(listTeams(repository, team.getName(), new ArrayList<>(), fields));
}

159
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java
View File

@ -13,8 +13,14 @@
package org.openmetadata.service.jdbi3;
import static org.openmetadata.common.utils.CommonUtil.listOf;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.csv.CsvUtil.addEntityReferences;
import static org.openmetadata.csv.CsvUtil.addField;
import static org.openmetadata.service.Entity.ROLE;
import static org.openmetadata.service.Entity.TEAM;
import static org.openmetadata.service.Entity.USER;
import java.io.IOException;
import java.util.ArrayList;
@ -26,6 +32,9 @@ import java.util.UUID;
import java.util.stream.Collectors;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.CSVRecord;
import org.openmetadata.csv.EntityCsv;
import org.openmetadata.schema.api.teams.CreateTeam.TeamType;
import org.openmetadata.schema.auth.SSOAuthMechanism;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
@ -34,6 +43,10 @@ import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.schema.type.csv.CsvDocumentation;
import org.openmetadata.schema.type.csv.CsvErrorType;
import org.openmetadata.schema.type.csv.CsvHeader;
import org.openmetadata.schema.type.csv.CsvImportResult;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.exception.CatalogExceptionMessage;
@ -55,14 +68,7 @@ public class UserRepository extends EntityRepository<User> {
private final EntityReference organization;
public UserRepository(CollectionDAO dao) {
super(
UserResource.COLLECTION_PATH,
Entity.USER,
User.class,
dao.userDAO(),
dao,
USER_PATCH_FIELDS,
USER_UPDATE_FIELDS);
super(UserResource.COLLECTION_PATH, USER, User.class, dao.userDAO(), dao, USER_PATCH_FIELDS, USER_UPDATE_FIELDS);
organization = dao.teamDAO().findEntityReferenceByName(Entity.ORGANIZATION_NAME, Include.ALL);
}
@ -152,6 +158,20 @@ public class UserRepository extends EntityRepository<User> {
return user.withInheritedRoles(fields.contains("roles") ? getInheritedRoles(user) : null);
}
@Override
public String exportToCsv(String importingTeam, String user) throws IOException {
Team team = daoCollection.teamDAO().findEntityByName(importingTeam);
return new UserCsv(team, user).exportCsv();
}
@Override
public CsvImportResult importFromCsv(String importingTeam, String csv, boolean dryRun, String user)
throws IOException {
Team team = daoCollection.teamDAO().findEntityByName(importingTeam);
UserCsv userCsv = new UserCsv(team, user);
return userCsv.importCsv(csv, dryRun);
}
public boolean isTeamJoinable(String teamId) throws IOException {
Team team = daoCollection.teamDAO().findEntityById(UUID.fromString(teamId), Include.NON_DELETED);
return team.getIsJoinable();
@ -204,7 +224,7 @@ public class UserRepository extends EntityRepository<User> {
private List<EntityReference> getOwns(User user) throws IOException {
// Compile entities owned by the user
List<EntityRelationshipRecord> ownedEntities =
daoCollection.relationshipDAO().findTo(user.getId().toString(), Entity.USER, Relationship.OWNS.ordinal());
daoCollection.relationshipDAO().findTo(user.getId().toString(), USER, Relationship.OWNS.ordinal());
// Compile entities owned by the team the user belongs to
List<EntityReference> teams = user.getTeams() == null ? getTeams(user) : user.getTeams();
@ -218,7 +238,7 @@ public class UserRepository extends EntityRepository<User> {
private List<EntityReference> getFollows(User user) throws IOException {
return EntityUtil.getEntityReferences(
daoCollection.relationshipDAO().findTo(user.getId().toString(), Entity.USER, Relationship.FOLLOWS.ordinal()));
daoCollection.relationshipDAO().findTo(user.getId().toString(), USER, Relationship.FOLLOWS.ordinal()));
}
private List<EntityReference> getTeamChildren(UUID teamId) throws IOException {
@ -252,13 +272,13 @@ public class UserRepository extends EntityRepository<User> {
/* Get all the roles that user has been assigned and inherited from the team to User entity */
private List<EntityReference> getRoles(User user) throws IOException {
List<EntityRelationshipRecord> roleIds = findTo(user.getId(), Entity.USER, Relationship.HAS, Entity.ROLE);
List<EntityRelationshipRecord> roleIds = findTo(user.getId(), USER, Relationship.HAS, Entity.ROLE);
return EntityUtil.populateEntityReferences(roleIds, Entity.ROLE);
}
/* Get all the teams that user belongs to User entity */
private List<EntityReference> getTeams(User user) throws IOException {
List<EntityRelationshipRecord> records = findFrom(user.getId(), Entity.USER, Relationship.HAS, Entity.TEAM);
List<EntityRelationshipRecord> records = findFrom(user.getId(), USER, Relationship.HAS, Entity.TEAM);
List<EntityReference> teams = EntityUtil.populateEntityReferences(records, Entity.TEAM);
teams = teams.stream().filter(team -> !team.getDeleted()).collect(Collectors.toList()); // Filter deleted teams
// If there are no teams that a user belongs to then return organization as the default team
@ -271,7 +291,7 @@ public class UserRepository extends EntityRepository<User> {
private void assignRoles(User user, List<EntityReference> roles) {
roles = listOrEmpty(roles);
for (EntityReference role : roles) {
addRelationship(user.getId(), role.getId(), Entity.USER, Entity.ROLE, Relationship.HAS);
addRelationship(user.getId(), role.getId(), USER, Entity.ROLE, Relationship.HAS);
}
}
@ -281,7 +301,7 @@ public class UserRepository extends EntityRepository<User> {
if (team.getId().equals(organization.getId())) {
continue; // Default relationship user to organization team is not stored
}
addRelationship(team.getId(), user.getId(), Entity.TEAM, Entity.USER, Relationship.HAS);
addRelationship(team.getId(), user.getId(), Entity.TEAM, USER, Relationship.HAS);
}
if (teams.size() > 1) {
// Remove organization team from the response
@ -290,6 +310,113 @@ public class UserRepository extends EntityRepository<User> {
}
}
public static class UserCsv extends EntityCsv<User> {
public static final CsvDocumentation DOCUMENTATION = getCsvDocumentation(USER);
public static final List<CsvHeader> HEADERS = DOCUMENTATION.getHeaders();
public final Team team;
UserCsv(Team importingTeam, String updatedBy) {
super(USER, HEADERS, updatedBy);
this.team = importingTeam;
}
@Override
protected User toEntity(CSVPrinter printer, CSVRecord record) throws IOException {
// Field 1, 2, 3, 4, 5, 6 - name, displayName, description, email, timezone, isAdmin
User user =
new User()
.withName(record.get(0))
.withDisplayName(record.get(1))
.withDescription(record.get(2))
.withEmail(record.get(3))
.withTimezone(record.get(4))
.withIsAdmin(getBoolean(printer, record, 5));
// Field 7 - team
user.setTeams(getTeams(printer, record, user.getName()));
if (!processRecord) {
return null;
}
// Field 8 - roles
user.setRoles(getEntityReferences(printer, record, 7, ROLE));
if (!processRecord) {
return null;
}
// TODO authentication mechanism?
return user;
}
@Override
protected List<String> toRecord(User entity) {
// Headers - name,displayName,description,email,timezone,isAdmin,team,roles
List<String> record = new ArrayList<>();
addField(record, entity.getName());
addField(record, entity.getDisplayName());
addField(record, entity.getDescription());
addField(record, entity.getEmail());
addField(record, entity.getTimezone());
addField(record, entity.getIsAdmin());
addField(record, entity.getTeams().get(0).getFullyQualifiedName());
addEntityReferences(record, entity.getRoles());
return record;
}
private List<User> listUsers(
TeamRepository teamRepository,
UserRepository userRepository,
String parentTeam,
List<User> users,
Fields fields)
throws IOException {
// Export the users by listing users for the entire team hierarchy
ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("team", parentTeam);
// Add users for the given team
List<User> userList = userRepository.listAll(fields, filter);
if (!nullOrEmpty(userList)) {
users.addAll(userList);
}
filter = new ListFilter(Include.NON_DELETED).addQueryParam("parentTeam", parentTeam);
List<Team> teamList = teamRepository.listAll(Fields.EMPTY_FIELDS, filter);
for (Team team : teamList) {
listUsers(teamRepository, userRepository, team.getName(), users, fields);
}
return users;
}
public String exportCsv() throws IOException {
UserRepository userRepository = (UserRepository) Entity.getEntityRepository(USER);
TeamRepository teamRepository = (TeamRepository) Entity.getEntityRepository(TEAM);
final Fields fields = userRepository.getFields("roles,teams");
return exportCsv(listUsers(teamRepository, userRepository, team.getName(), new ArrayList<>(), fields));
}
private List<EntityReference> getTeams(CSVPrinter printer, CSVRecord record, String user) throws IOException {
List<EntityReference> teams = getEntityReferences(printer, record, 6, Entity.TEAM);
// Validate team being created is under the hierarchy of the team for which CSV is being imported to
for (EntityReference teamRef : listOrEmpty(teams)) {
if (teamRef.getName().equals(team.getName())) {
continue; // Team is same as the team to which CSV is being imported, then it is in the same hierarchy
}
// Else the parent should already exist
if (!SubjectCache.getInstance().isInTeam(team.getName(), listOf(teamRef))) {
importFailure(printer, invalidTeam(6, team.getName(), user, teamRef.getName()), record);
processRecord = false;
}
}
return teams;
}
public static String invalidTeam(int field, String team, String user, String userTeam) {
String error = String.format("Team %s of user %s is not under %s team hierarchy", userTeam, user, team);
return String.format("#%s: Field %d error - %s", CsvErrorType.INVALID_FIELD, field + 1, error);
}
}
/** Handles entity updated from PUT and POST operation. */
public class UserUpdater extends EntityUpdater {
public UserUpdater(User original, User updated, Operation operation) {
@ -311,7 +438,7 @@ public class UserRepository extends EntityRepository<User> {
private void updateRoles(User original, User updated) throws IOException {
// Remove roles from original and add roles from updated
deleteFrom(original.getId(), Entity.USER, Relationship.HAS, Entity.ROLE);
deleteFrom(original.getId(), USER, Relationship.HAS, Entity.ROLE);
assignRoles(updated, updated.getRoles());
List<EntityReference> origRoles = listOrEmpty(original.getRoles());
@ -327,7 +454,7 @@ public class UserRepository extends EntityRepository<User> {
private void updateTeams(User original, User updated) throws IOException {
// Remove teams from original and add teams from updated
deleteTo(original.getId(), Entity.USER, Relationship.HAS, Entity.TEAM);
deleteTo(original.getId(), USER, Relationship.HAS, Entity.TEAM);
assignTeams(updated, updated.getTeams());
List<EntityReference> origTeams = listOrEmpty(original.getTeams());

7
openmetadata-service/src/main/java/org/openmetadata/service/resources/alerts/AlertResource.java
View File

@ -66,9 +66,9 @@ import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.alerts.ActivityFeedAlertCache;
import org.openmetadata.service.alerts.AlertUtil;
import org.openmetadata.service.alerts.AlertsPublisherManager;
import org.openmetadata.service.jdbi3.AlertActionRepository;
import org.openmetadata.service.jdbi3.AlertRepository;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
@ -122,9 +122,10 @@ public class AlertResource extends EntityResource<Alert, AlertRepository> {
activityFeedAlert = JsonUtils.readObjects(alertJson, Alert.class).get(0);
activityFeedAlert.setId(UUID.randomUUID());
// populate alert actions
EntityRepository<AlertAction> actionEntityRepository = Entity.getEntityRepository(Entity.ALERT_ACTION);
AlertActionRepository alertActionRepository =
(AlertActionRepository) Entity.getEntityRepository(Entity.ALERT_ACTION);
AlertAction action =
actionEntityRepository.getByName(null, alertActions.getName(), actionEntityRepository.getFields("id"));
alertActionRepository.getByName(null, alertActions.getName(), alertActionRepository.getFields("id"));
activityFeedAlert.setAlertActions(List.of(action.getEntityReference()));
dao.initializeEntity(activityFeedAlert);
} catch (Exception e) {

6
openmetadata-service/src/main/java/org/openmetadata/service/resources/permissions/PermissionsResource.java
View File

@ -141,7 +141,7 @@ public class PermissionsResource {
@Parameter(description = "Resource type", schema = @Schema(type = "String")) @PathParam("resource")
String resource,
@Parameter(description = "Entity Id", schema = @Schema(type = "UUID")) @PathParam("id") UUID id) {
EntityRepository<EntityInterface> entityRepository = Entity.getEntityRepository(resource);
EntityRepository<? extends EntityInterface> entityRepository = Entity.getEntityRepository(resource);
ResourceContext resourceContext =
ResourceContext.builder().resource(resource).id(id).entityRepository(entityRepository).build();
return authorizer.getPermission(securityContext, user, resourceContext);
@ -174,7 +174,7 @@ public class PermissionsResource {
@Parameter(description = "Resource type", schema = @Schema(type = "String")) @PathParam("resource")
String resource,
@Parameter(description = "Entity Name", schema = @Schema(type = "String")) @PathParam("name") String name) {
EntityRepository<EntityInterface> entityRepository = Entity.getEntityRepository(resource);
EntityRepository<? extends EntityInterface> entityRepository = Entity.getEntityRepository(resource);
ResourceContext resourceContext =
ResourceContext.builder().resource(resource).name(name).entityRepository(entityRepository).build();
return authorizer.getPermission(securityContext, user, resourceContext);
@ -202,7 +202,7 @@ public class PermissionsResource {
throws IOException {
// User must have read access to policies
OperationContext operationContext = new OperationContext(Entity.POLICY, MetadataOperation.VIEW_ALL);
EntityRepository<EntityInterface> dao = Entity.getEntityRepository(Entity.POLICY);
EntityRepository<? extends EntityInterface> dao = Entity.getEntityRepository(Entity.POLICY);
for (UUID id : ids) {
ResourceContext resourceContext = EntityResource.getResourceContext(Entity.POLICY, dao).id(id).build();
authorizer.authorize(securityContext, operationContext, resourceContext);

23
openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelCache.java
View File

@ -30,7 +30,10 @@ import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.TagLabel.TagSource;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.ClassificationRepository;
import org.openmetadata.service.jdbi3.GlossaryRepository;
import org.openmetadata.service.jdbi3.GlossaryTermRepository;
import org.openmetadata.service.jdbi3.TagRepository;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.FullyQualifiedName;
@ -42,13 +45,13 @@ public class TagLabelCache {
private static final TagLabelCache INSTANCE = new TagLabelCache();
private static volatile boolean INITIALIZED = false;
protected static EntityRepository<Tag> TAG_REPOSITORY;
protected static EntityRepository<Classification> TAG_CATEGORY_REPOSITORY;
protected static TagRepository TAG_REPOSITORY;
protected static ClassificationRepository TAG_CLASSIFICATION_REPOSITORY;
protected static LoadingCache<String, Tag> TAG_CACHE; // Tag fqn to Tag
protected static LoadingCache<String, Classification> TAG_CATEGORY_CACHE; // Classification name to Classification
protected static EntityRepository<GlossaryTerm> GLOSSARY_TERM_REPOSITORY;
protected static EntityRepository<Glossary> GLOSSARY_REPOSITORY;
protected static GlossaryTermRepository GLOSSARY_TERM_REPOSITORY;
protected static GlossaryRepository GLOSSARY_REPOSITORY;
protected static LoadingCache<String, GlossaryTerm> GLOSSARY_TERM_CACHE; // Glossary term fqn to GlossaryTerm
protected static LoadingCache<String, Glossary> GLOSSARY_CACHE; // Glossary fqn to Glossary
@ -62,8 +65,8 @@ public class TagLabelCache {
.build(new ClassificationLoader());
TAG_CACHE =
CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(2, TimeUnit.MINUTES).build(new TagLoader());
TAG_REPOSITORY = Entity.getEntityRepository(Entity.TAG);
TAG_CATEGORY_REPOSITORY = Entity.getEntityRepository(Entity.CLASSIFICATION);
TAG_REPOSITORY = (TagRepository) Entity.getEntityRepository(Entity.TAG);
TAG_CLASSIFICATION_REPOSITORY = (ClassificationRepository) Entity.getEntityRepository(Entity.CLASSIFICATION);
GLOSSARY_CACHE =
CacheBuilder.newBuilder().maximumSize(25).expireAfterWrite(2, TimeUnit.MINUTES).build(new GlossaryLoader());
@ -72,8 +75,8 @@ public class TagLabelCache {
.maximumSize(100)
.expireAfterWrite(2, TimeUnit.MINUTES)
.build(new GlossaryTermLoader());
GLOSSARY_TERM_REPOSITORY = Entity.getEntityRepository(Entity.GLOSSARY_TERM);
GLOSSARY_REPOSITORY = Entity.getEntityRepository(Entity.GLOSSARY);
GLOSSARY_TERM_REPOSITORY = (GlossaryTermRepository) Entity.getEntityRepository(Entity.GLOSSARY_TERM);
GLOSSARY_REPOSITORY = (GlossaryRepository) Entity.getEntityRepository(Entity.GLOSSARY);
INITIALIZED = true;
} else {
LOG.info("Subject cache is already initialized");
@ -156,7 +159,7 @@ public class TagLabelCache {
static class ClassificationLoader extends CacheLoader<String, Classification> {
@Override
public Classification load(@CheckForNull String categoryName) throws IOException {
Classification category = TAG_CATEGORY_REPOSITORY.getByName(null, categoryName, Fields.EMPTY_FIELDS);
Classification category = TAG_CLASSIFICATION_REPOSITORY.getByName(null, categoryName, Fields.EMPTY_FIELDS);
LOG.info("Loaded user {}:{}", category.getName(), category.getId());
return category;
}

8
openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java
View File

@ -52,7 +52,6 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.api.classification.CreateTag;
import org.openmetadata.schema.api.classification.LoadTags;
import org.openmetadata.schema.api.data.RestoreEntity;
@ -64,6 +63,7 @@ import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.jdbi3.ClassificationRepository;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.ListFilter;
@ -103,7 +103,8 @@ public class TagResource extends EntityResource<Tag, TagRepository> {
if (!(daoCollection.relationshipDAO().findIfAnyRelationExist(CLASSIFICATION, TAG) > 0)) {
// We are missing relationship for classification -> tag, and also tag -> tag (parent relationship)
// Find tag definitions and load classifications from the json file, if necessary
EntityRepository<Classification> classificationRepository = Entity.getEntityRepository(CLASSIFICATION);
ClassificationRepository classificationRepository =
(ClassificationRepository) Entity.getEntityRepository(CLASSIFICATION);
try {
List<Classification> classificationList =
classificationRepository.listAll(classificationRepository.getFields("*"), new ListFilter(Include.ALL));
@ -154,7 +155,8 @@ public class TagResource extends EntityResource<Tag, TagRepository> {
// TODO: Once we have migrated to the version above 0.13.1, then this can be removed
migrateTags();
// Find tag definitions and load classifications from the json file, if necessary
EntityRepository<EntityInterface> classificationRepository = Entity.getEntityRepository(CLASSIFICATION);
ClassificationRepository classificationRepository =
(ClassificationRepository) Entity.getEntityRepository(CLASSIFICATION);
List<LoadTags> loadTagsList =
EntityRepository.getEntitiesFromSeedData(CLASSIFICATION, ".*json/data/tags/.*\\.json$", LoadTags.class);
for (LoadTags loadTags : loadTagsList) {

6
openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java
View File

@ -48,7 +48,6 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.api.data.RestoreEntity;
import org.openmetadata.schema.api.teams.CreateRole;
import org.openmetadata.schema.entity.teams.Role;
@ -58,7 +57,6 @@ import org.openmetadata.schema.type.Include;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.RoleRepository;
import org.openmetadata.service.resources.Collection;
@ -436,7 +434,7 @@ public class RoleResource extends EntityResource<Role, RoleRepository> {
}
public static EntityReference getRole(String roleName) {
EntityRepository<EntityInterface> dao = Entity.getEntityRepository(Entity.ROLE);
return dao.dao.findEntityReferenceByName(roleName);
RoleRepository roleRepository = (RoleRepository) Entity.getEntityRepository(Entity.ROLE);
return roleRepository.dao.findEntityReferenceByName(roleName);
}
}

5
openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java
View File

@ -457,7 +457,10 @@ public class TeamResource extends EntityResource<Team, TeamRepository> {
@GET
@Path("/documentation/csv")
@Valid
@Operation(operationId = "getCsvDocumentation", summary = "Get CSV documentation", tags = "glossaries")
@Operation(
operationId = "getCsvDocumentation",
summary = "Get CSV documentation for team import/export",
tags = "teams")
public String getCsvDocumentation(@Context SecurityContext securityContext, @PathParam("name") String name)
throws IOException {
return JsonUtils.pojoToJson(TeamCsv.DOCUMENTATION);

75
openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java
View File

@ -93,6 +93,7 @@ import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.schema.type.ProviderType;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.schema.type.csv.CsvImportResult;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.exception.CatalogExceptionMessage;
@ -101,6 +102,7 @@ import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.TokenRepository;
import org.openmetadata.service.jdbi3.UserRepository;
import org.openmetadata.service.jdbi3.UserRepository.UserCsv;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.secrets.SecretsManager;
@ -1027,6 +1029,79 @@ public class UserResource extends EntityResource<User, UserRepository> {
return Response.status(Response.Status.OK).entity(authHandler.getNewAccessToken(refreshRequest)).build();
}
@GET
@Path("/documentation/csv")
@Valid
@Operation(
operationId = "getCsvDocumentation",
summary = "Get CSV documentation for user import/export",
tags = "users")
public String getUserCsvDocumentation(@Context SecurityContext securityContext, @PathParam("name") String name)
throws IOException {
return JsonUtils.pojoToJson(UserCsv.DOCUMENTATION);
}
@GET
@Path("/export")
@Produces(MediaType.TEXT_PLAIN)
@Valid
@Operation(
operationId = "exportUsers",
summary = "Export users in a team in CSV format",
tags = "users",
responses = {
@ApiResponse(
responseCode = "200",
description = "Exported csv with user information",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = String.class)))
})
public String exportUsersCsv(
@Context SecurityContext securityContext,
@Parameter(
description = "Name of the team to under which the users are imported to",
required = true,
schema = @Schema(type = "string"))
@QueryParam("team")
String team)
throws IOException {
return exportCsvInternal(securityContext, team);
}
@PUT
@Path("/import")
@Consumes(MediaType.TEXT_PLAIN)
@Valid
@Operation(
operationId = "importTeams",
summary = "Import from CSV to create, and update teams.",
tags = "users",
responses = {
@ApiResponse(
responseCode = "200",
description = "Import result",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = CsvImportResult.class)))
})
public CsvImportResult importCsv(
@Context SecurityContext securityContext,
@Parameter(
description = "Name of the team to under which the users are imported to",
required = true,
schema = @Schema(type = "string"))
@QueryParam("team")
String team,
@Parameter(
description =
"Dry-run when true is used for validating the CSV without really importing it. (default=true)",
schema = @Schema(type = "boolean"))
@DefaultValue("true")
@QueryParam("dryRun")
boolean dryRun,
String csv)
throws IOException {
return importCsvInternal(securityContext, team, csv, dryRun);
}
private User getUser(SecurityContext securityContext, CreateUser create) {
return new User()
.withId(UUID.randomUUID())

12
openmetadata-service/src/main/java/org/openmetadata/service/secrets/SecretsManagerUpdateService.java
View File

@ -28,9 +28,10 @@ import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.SecretsManagerUpdateException;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.IngestionPipelineRepository;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.jdbi3.ServiceEntityRepository;
import org.openmetadata.service.jdbi3.UserRepository;
import org.openmetadata.service.resources.CollectionRegistry;
import org.openmetadata.service.resources.CollectionRegistry.CollectionDetails;
import org.openmetadata.service.resources.services.ServiceEntityResource;
@ -49,8 +50,8 @@ import org.openmetadata.service.util.EntityUtil;
public class SecretsManagerUpdateService {
private final SecretsManager secretManager;
private final SecretsManager oldSecretManager;
private final EntityRepository<User> userRepository;
private final EntityRepository<IngestionPipeline> ingestionPipelineRepository;
private final UserRepository userRepository;
private final IngestionPipelineRepository ingestionPipelineRepository;
private final Map<Class<? extends ServiceConnectionEntityInterface>, ServiceEntityRepository<?, ?>>
connectionTypeRepositoriesMap;
@ -58,8 +59,9 @@ public class SecretsManagerUpdateService {
public SecretsManagerUpdateService(SecretsManager secretsManager, String clusterName) {
this.secretManager = secretsManager;
this.connectionTypeRepositoriesMap = retrieveConnectionTypeRepositoriesMap();
this.userRepository = Entity.getEntityRepository(Entity.USER);
this.ingestionPipelineRepository = Entity.getEntityRepository(Entity.INGESTION_PIPELINE);
this.userRepository = (UserRepository) Entity.getEntityRepository(Entity.USER);
this.ingestionPipelineRepository =
(IngestionPipelineRepository) Entity.getEntityRepository(Entity.INGESTION_PIPELINE);
// by default, it is going to be non-managed secrets manager since decrypt is the same for all of them
this.oldSecretManager = SecretsManagerFactory.createSecretsManager(null, clusterName);
}

4
openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java
View File

@ -25,7 +25,7 @@ import org.openmetadata.schema.type.ResourcePermission;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.UserRepository;
import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.security.policyevaluator.PolicyEvaluator;
import org.openmetadata.service.security.policyevaluator.ResourceContextInterface;
@ -84,7 +84,7 @@ public class NoopAuthorizer implements Authorizer {
private void addOrUpdateUser(User user) {
try {
EntityRepository<User> userRepository = Entity.getEntityRepository(Entity.USER);
UserRepository userRepository = (UserRepository) Entity.getEntityRepository(Entity.USER);
RestUtil.PutResponse<User> addedUser = userRepository.createOrUpdate(null, user);
LOG.debug("Added anonymous user entry: {}", addedUser);
} catch (IOException exception) {

6
openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/PolicyCache.java
View File

@ -28,7 +28,7 @@ import org.openmetadata.schema.entity.policies.Policy;
import org.openmetadata.schema.entity.policies.accessControl.Rule;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.PolicyRepository;
import org.openmetadata.service.util.EntityUtil.Fields;
/** Subject context used for Access Control Policies */
@ -38,7 +38,7 @@ public class PolicyCache {
private static volatile boolean INITIALIZED = false;
protected static LoadingCache<UUID, List<CompiledRule>> POLICY_CACHE;
private static EntityRepository<Policy> POLICY_REPOSITORY;
private static PolicyRepository POLICY_REPOSITORY;
private static Fields FIELDS;
public static PolicyCache getInstance() {
@ -49,7 +49,7 @@ public class PolicyCache {
public static void initialize() {
if (!INITIALIZED) {
POLICY_CACHE = CacheBuilder.newBuilder().maximumSize(100).build(new PolicyLoader());
POLICY_REPOSITORY = Entity.getEntityRepository(Entity.POLICY);
POLICY_REPOSITORY = (PolicyRepository) Entity.getEntityRepository(Entity.POLICY);
FIELDS = POLICY_REPOSITORY.getFields("rules");
INITIALIZED = true;
}

6
openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/RoleCache.java
View File

@ -25,7 +25,7 @@ import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.entity.teams.Role;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.RoleRepository;
import org.openmetadata.service.util.EntityUtil.Fields;
/** Subject context used for Access Control Policies */
@ -34,7 +34,7 @@ public class RoleCache {
private static final RoleCache INSTANCE = new RoleCache();
private static volatile boolean INITIALIZED = false;
protected static LoadingCache<UUID, Role> ROLE_CACHE;
private static EntityRepository<Role> ROLE_REPOSITORY;
private static RoleRepository ROLE_REPOSITORY;
private static Fields FIELDS;
public static RoleCache getInstance() {
@ -45,7 +45,7 @@ public class RoleCache {
public static void initialize() {
if (!INITIALIZED) {
ROLE_CACHE = CacheBuilder.newBuilder().maximumSize(100).build(new RoleLoader());
ROLE_REPOSITORY = Entity.getEntityRepository(Entity.ROLE);
ROLE_REPOSITORY = (RoleRepository) Entity.getEntityRepository(Entity.ROLE);
FIELDS = ROLE_REPOSITORY.getFields("policies");
INITIALIZED = true;
}

11
openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/SubjectCache.java
View File

@ -35,7 +35,8 @@ import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.TeamRepository;
import org.openmetadata.service.jdbi3.UserRepository;
import org.openmetadata.service.util.EntityUtil.Fields;
/** Subject context used for Access Control Policies */
@ -46,9 +47,9 @@ public class SubjectCache {
protected static LoadingCache<String, SubjectContext> USER_CACHE;
protected static LoadingCache<UUID, SubjectContext> USER_CACHE_WIH_ID;
protected static LoadingCache<UUID, Team> TEAM_CACHE;
protected static EntityRepository<User> USER_REPOSITORY;
protected static UserRepository USER_REPOSITORY;
protected static Fields USER_FIELDS;
protected static EntityRepository<Team> TEAM_REPOSITORY;
protected static TeamRepository TEAM_REPOSITORY;
protected static Fields TEAM_FIELDS;
// Expected to be called only once from the DefaultAuthorizer
@ -63,9 +64,9 @@ public class SubjectCache {
.build(new UserLoaderWithId());
TEAM_CACHE =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new TeamLoader());
USER_REPOSITORY = Entity.getEntityRepository(Entity.USER);
USER_REPOSITORY = (UserRepository) Entity.getEntityRepository(Entity.USER);
USER_FIELDS = USER_REPOSITORY.getFields("roles, teams, isAdmin");
TEAM_REPOSITORY = Entity.getEntityRepository(Entity.TEAM);
TEAM_REPOSITORY = (TeamRepository) Entity.getEntityRepository(Entity.TEAM);
TEAM_FIELDS = TEAM_REPOSITORY.getFields("defaultRoles, policies, parents");
INSTANCE = new SubjectCache();
INITIALIZED = true;

5
openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/TestCaseResourceContext.java
View File

@ -26,6 +26,7 @@ import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.TestCaseRepository;
import org.openmetadata.service.resources.feeds.MessageParser.EntityLink;
import org.openmetadata.service.util.EntityUtil;
@ -91,14 +92,14 @@ public class TestCaseResourceContext implements ResourceContextInterface {
}
private static EntityInterface resolveEntityById(UUID id) throws IOException {
EntityRepository<TestCase> dao = Entity.getEntityRepository(Entity.TEST_CASE);
TestCaseRepository dao = (TestCaseRepository) Entity.getEntityRepository(Entity.TEST_CASE);
TestCase testCase = dao.get(null, id, dao.getFields("entityLink"), Include.ALL);
return resolveEntityByEntityLink(EntityLink.parse(testCase.getEntityLink()));
}
private static EntityInterface resolveEntityByName(String fqn) throws IOException {
if (fqn == null) return null;
EntityRepository<TestCase> dao = Entity.getEntityRepository(Entity.TEST_CASE);
TestCaseRepository dao = (TestCaseRepository) Entity.getEntityRepository(Entity.TEST_CASE);
TestCase testCase = dao.getByName(null, fqn, dao.getFields("entityLink"), Include.ALL);
return resolveEntityByEntityLink(EntityLink.parse(testCase.getEntityLink()));
}

6
openmetadata-service/src/main/java/org/openmetadata/service/util/ElasticSearchIndexUtil.java
View File

@ -257,10 +257,10 @@ public class ElasticSearchIndexUtil {
indexType);
} else {
// Start fetching a list of Entities and pushing them to ES
EntityRepository<EntityInterface> entityRepository = Entity.getEntityRepository(entityType);
EntityRepository<? extends EntityInterface> entityRepository = Entity.getEntityRepository(entityType);
List<String> allowedFields = entityRepository.getAllowedFields();
String fields = String.join(",", allowedFields);
ResultList<EntityInterface> result;
ResultList<? extends EntityInterface> result;
String after = null;
try {
do {
@ -309,7 +309,7 @@ public class ElasticSearchIndexUtil {
ElasticSearchIndexDefinition.ElasticSearchIndexType indexType,
BulkProcessor bulkProcessor,
String entityType,
List<EntityInterface> entities) {
List<? extends EntityInterface> entities) {
for (EntityInterface entity : entities) {
if (entityType.equals(TABLE)) {
((Table) entity).getColumns().forEach(table -> table.setProfile(null));

4
openmetadata-service/src/main/java/org/openmetadata/service/util/NotificationHandler.java
View File

@ -39,7 +39,7 @@ import org.openmetadata.schema.type.Post;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.UserRepository;
import org.openmetadata.service.resources.feeds.MessageParser;
import org.openmetadata.service.resources.settings.SettingsCache;
import org.openmetadata.service.socket.WebSocketManager;
@ -160,7 +160,7 @@ public class NotificationHandler {
}
private void handleEmailNotifications(HashSet<UUID> userList, Thread thread) {
EntityRepository<User> repository = Entity.getEntityRepository(USER);
UserRepository repository = (UserRepository) Entity.getEntityRepository(USER);
URI urlInstance = thread.getHref();
userList.forEach(
(id) -> {

4
openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java
View File

@ -62,7 +62,7 @@ public final class UserUtil {
}
public static void addUserForBasicAuth(String username, String pwd, String domain) throws IOException {
EntityRepository<User> userRepository = Entity.getEntityRepository(Entity.USER);
UserRepository userRepository = (UserRepository) Entity.getEntityRepository(Entity.USER);
User originalUser;
try {
List<String> fields = userRepository.getAllowedFieldsCopy();
@ -97,7 +97,7 @@ public final class UserUtil {
}
public static User addOrUpdateUser(User user) {
EntityRepository<User> userRepository = Entity.getEntityRepository(Entity.USER);
UserRepository userRepository = (UserRepository) Entity.getEntityRepository(Entity.USER);
try {
RestUtil.PutResponse<User> addedUser = userRepository.createOrUpdate(null, user);
// should not log the user auth details in LOGS

76
openmetadata-service/src/main/resources/json/data/user/userCsvDocumentation.json Normal file
View File

@ -0,0 +1,76 @@
{
"summary": "Documentation for CSV file used for importing and exporting users under a team. Users can be imported to any team that is children of the team for which CSV is imported for.",
"headers": [
{
"name": "name",
"required": true,
"description": "The name of the user being created. This is same as the login name.",
"examples": [
"`john`",
"`adam.smith`"
]
},
{
"name": "displayName",
"required": false,
"description": "Display name for the user.",
"examples": [
"`John`",
"`Adam Smith`"
]
},
{
"name": "description",
"required": false,
"description": "Description for the user in markdown format.",
"examples": [
"`John` is a Data Scientist."
]
},
{
"name": "email",
"required": true,
"description": "Email address of the user.",
"examples": [
"`john@company.com`"
]
},
{
"name": "timezone",
"required": false,
"description": "The timezone of the user.",
"examples": [
"`America/Los_Angeles`",
"`Brazil/East`"
]
},
{
"name": "isAdmin",
"required": false,
"description": "Set true if the user is an Admin. Default - false",
"examples": [
"`true` or `false`",
"\"\" will use default value `false`"
]
},
{
"name": "teams",
"required": true,
"description": "Teams the user belongs to. For multiple teams, provide team names separated by ';'",
"examples": [
"Marketing team",
"Marketing team;Finance team"
]
},
{
"name": "Roles",
"required": false,
"description": "Roles that are assigned to a user.",
"examples": [
"`Data consumer`",
"`Data consumer;Data steward`",
"`\"\" for no role`"
]
}
]
}

3
openmetadata-service/src/test/java/org/openmetadata/csv/CsvUtilTest.java
View File

@ -74,7 +74,8 @@ public class CsvUtilTest {
// Break a csv text into records, sort it and compare
List<String> expectedCsvRecords = listOf(expectedCsv.split(CsvUtil.LINE_SEPARATOR));
List<String> actualCsvRecords = listOf(actualCsv.split(CsvUtil.LINE_SEPARATOR));
assertEquals(expectedCsvRecords.size(), actualCsvRecords.size());
assertEquals(
expectedCsvRecords.size(), actualCsvRecords.size(), "Expected " + expectedCsv + " actual " + actualCsv);
Collections.sort(expectedCsvRecords);
Collections.sort(actualCsvRecords);
for (int i = 0; i < expectedCsvRecords.size(); i++) {

2
openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java
View File

@ -90,7 +90,7 @@ public class EntityCsvTest {
int expectedRowsProcessed,
int expectedRowsPassed,
int expectedRowsFailed) {
assertEquals(expectedStatus, importResult.getStatus(), importResult.getImportResultsCsv());
assertEquals(expectedStatus, importResult.getStatus(), importResult.toString());
assertEquals(expectedRowsProcessed, importResult.getNumberOfRowsProcessed());
assertEquals(expectedRowsPassed, importResult.getNumberOfRowsPassed());
assertEquals(expectedRowsFailed, importResult.getNumberOfRowsFailed());

4
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -2027,7 +2027,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
Awaitility.await("Wait for expected change event at timestamp " + timestamp)
.pollInterval(Duration.ofMillis(100L))
.atMost(Duration.ofMillis(10 * 100L)) // 10 iterations
.atMost(Duration.ofMillis(20 * 100L)) // 10 iterations
.until(
() ->
eventHolder.hasExpectedEvent(
@ -2379,7 +2379,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
return String.join(",", Entity.getAllowedFields(entityClass));
}
protected CsvImportResult importCsv(String entityName, String csv, boolean dryRun) throws HttpResponseException {
public CsvImportResult importCsv(String entityName, String csv, boolean dryRun) throws HttpResponseException {
WebTarget target = getResourceByName(entityName + "/import");
target = !dryRun ? target.queryParam("dryRun", false) : target;
return TestUtils.putCsv(target, csv, CsvImportResult.class, Status.OK, ADMIN_AUTH_HEADERS);

8
openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java
View File

@ -74,6 +74,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.csv.EntityCsv;
import org.openmetadata.csv.EntityCsvTest;
import org.openmetadata.schema.api.policies.CreatePolicy;
import org.openmetadata.schema.api.teams.CreateRole;
import org.openmetadata.schema.api.teams.CreateTeam;
@ -752,6 +753,13 @@ public class TeamResourceTest extends EntityResourceTest<Team, CreateTeam> {
String record3 = getRecord(3, GROUP, team.getName(), null, true, null, (List<Policy>) null);
List<String> newRecords = listOf(record3);
testImportExport(team.getName(), TeamCsv.HEADERS, createRecords, updateRecords, newRecords);
// Import to team111 a user with parent team1 - since team1 is not under team111 hierarchy, import should fail
String record4 = getRecord(3, GROUP, "x1", null, true, null, (List<Policy>) null);
String csv = EntityCsvTest.createCsv(TeamCsv.HEADERS, listOf(record4), null);
CsvImportResult result = importCsv("x111", csv, false);
String error = TeamCsv.invalidTeam(4, "x111", "x3", "x1");
assertTrue(result.getImportResultsCsv().contains(error));
}
private static void validateTeam(

115
openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java
View File

@ -30,6 +30,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.openmetadata.common.utils.CommonUtil.listOf;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.csv.CsvUtil.recordToString;
import static org.openmetadata.csv.EntityCsvTest.assertRows;
import static org.openmetadata.csv.EntityCsvTest.assertSummary;
import static org.openmetadata.csv.EntityCsvTest.createCsv;
import static org.openmetadata.csv.EntityCsvTest.getFailedRecord;
import static org.openmetadata.service.exception.CatalogExceptionMessage.PASSWORD_INVALID_FORMAT;
import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound;
import static org.openmetadata.service.exception.CatalogExceptionMessage.notAdmin;
@ -71,6 +76,8 @@ import java.util.TimeZone;
import java.util.UUID;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response.Status;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.client.HttpResponseException;
@ -78,6 +85,8 @@ import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.TestMethodOrder;
import org.openmetadata.csv.EntityCsv;
import org.openmetadata.csv.EntityCsvTest;
import org.openmetadata.schema.api.CreateBot;
import org.openmetadata.schema.api.teams.CreateUser;
import org.openmetadata.schema.auth.GenerateTokenRequest;
@ -99,9 +108,12 @@ import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.ImageList;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.schema.type.Profile;
import org.openmetadata.schema.type.csv.CsvImportResult;
import org.openmetadata.service.Entity;
import org.openmetadata.service.auth.JwtResponse;
import org.openmetadata.service.exception.CatalogExceptionMessage;
import org.openmetadata.service.jdbi3.TeamRepository.TeamCsv;
import org.openmetadata.service.jdbi3.UserRepository.UserCsv;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.bots.BotResourceTest;
import org.openmetadata.service.resources.databases.TableResourceTest;
@ -466,10 +478,6 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
assertEquals(user1, users.getData().get(0));
}
private CreateUser createBotUserRequest(TestInfo test, int index) {
return createBotUserRequest(getEntityName(test, index));
}
@Test
void get_listUsersWithTeamsPagination(TestInfo test) throws IOException {
TeamResourceTest teamResourceTest = new TeamResourceTest();
@ -532,6 +540,12 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
assertEquals(user1, users.getData().get(0));
}
@Test
void get_generateRandomPassword() throws HttpResponseException {
String randomPwd = TestUtils.get(getResource("users/generateRandomPwd"), String.class, ADMIN_AUTH_HEADERS);
assertDoesNotThrow(() -> PasswordUtil.validatePassword(randomPwd), PASSWORD_INVALID_FORMAT);
}
/**
* @see EntityResourceTest put_addDeleteFollower_200 test for tests related to GET user with owns field parameter
* @see EntityResourceTest put_addDeleteFollower_200 for tests related getting user with follows list
@ -770,12 +784,6 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
assertEquals(StringUtils.EMPTY, jwtAuthMechanism.getJWTToken());
}
@Test
void get_generateRandomPassword() throws HttpResponseException {
String randomPwd = TestUtils.get(getResource("users/generateRandomPwd"), String.class, ADMIN_AUTH_HEADERS);
assertDoesNotThrow(() -> PasswordUtil.validatePassword(randomPwd), PASSWORD_INVALID_FORMAT);
}
@Test
void post_createUser_BasicAuth_AdminCreate_login_200_ok(TestInfo test) throws HttpResponseException {
// Create a user with Auth and Try Logging in
@ -887,6 +895,74 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
CatalogExceptionMessage.INVALID_USERNAME_PASSWORD);
}
@Test
void testCsvDocumentation() throws HttpResponseException {
assertEquals(UserCsv.DOCUMENTATION, getCsvDocumentation());
}
@Test
void testImportInvalidCsv() throws IOException {
// Headers - name,displayName,description,email,timezone,isAdmin,teams,roles
TeamResourceTest teamResourceTest = new TeamResourceTest();
Team team = teamResourceTest.createEntity(teamResourceTest.createRequest("team-invalidCsv"), ADMIN_AUTH_HEADERS);
// Invalid team
String resultsHeader = recordToString(EntityCsv.getResultHeaders(UserCsv.HEADERS));
String record = "user,,,user@domain.com,,,invalidTeam,";
String csv = createCsv(UserCsv.HEADERS, listOf(record), null);
CsvImportResult result = importCsv(team.getName(), csv, false);
assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1);
String[] expectedRows = {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(6, "invalidTeam"))};
assertRows(result, expectedRows);
// Invalid roles
record = "user,,,user@domain.com,,,team-invalidCsv,invalidRole";
csv = createCsv(UserCsv.HEADERS, listOf(record), null);
result = importCsv(team.getName(), csv, false);
assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1);
expectedRows = new String[] {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(7, "invalidRole"))};
assertRows(result, expectedRows);
}
@Test
void testUserImportExport() throws IOException {
// Create team hierarchy - team with children t1, t1 has t11
// "name", "displayName", "description", "teamType", "parents", "owner", "isJoinable", "defaultRoles", & "policies"
TeamResourceTest teamResourceTest = new TeamResourceTest();
String team = "teamImportExport,,,Division,Organization,,,,";
String team1 = "teamImportExport1,,,Department,teamImportExport,,,,";
String team11 = "teamImportExport11,,,Group,teamImportExport1,,,,";
String csv = EntityCsvTest.createCsv(TeamCsv.HEADERS, listOf(team, team1, team11), null);
CsvImportResult result = teamResourceTest.importCsv(ORG_TEAM.getName(), csv, false);
assertEquals(0, result.getNumberOfRowsFailed());
// Create users in the team hierarchy
// Headers - name,displayName,description,email,timezone,isAdmin,teams,roles
String user = "userImportExport,d,s,userImportExport@domain.com,America/Los_Angeles,true,teamImportExport,";
String user1 = "userImportExport1,,,userImportExport1@domain.com,,,teamImportExport1,DataConsumer";
String user11 = "userImportExport11,,,userImportExport11@domain.com,,,teamImportExport11,";
List<String> createRecords = listOf(user, user1, user11);
// Update user descriptions
user = "userImportExport,displayName,,userImportExport@domain.com,,,teamImportExport,";
user1 = "userImportExport1,displayName1,,userImportExport1@domain.com,,,teamImportExport1,";
user11 = "userImportExport11,displayName11,,userImportExport11@domain.com,,,teamImportExport11,";
List<String> updateRecords = listOf(user, user1, user11);
// Add new users
String user2 = "userImportExport2,displayName2,,userImportExport2@domain.com,,,teamImportExport1,";
String user21 = "userImportExport21,displayName21,,userImportExport11@domain.com,,,teamImportExport11,";
List<String> newRecords = listOf(user2, user21);
testImportExport("teamImportExport", UserCsv.HEADERS, createRecords, updateRecords, newRecords);
// Import to team11 a user in team1 - since team1 is not under team11 hierarchy, import should fail
String user3 = "userImportExport3,displayName3,,userImportExport3@domain.com,,,teamImportExport1,";
csv = EntityCsvTest.createCsv(UserCsv.HEADERS, listOf(user3), null);
result = importCsv("teamImportExport11", csv, false);
String error = UserCsv.invalidTeam(6, "teamImportExport11", "userImportExport3", "teamImportExport1");
assertTrue(result.getImportResultsCsv().contains(error));
}
private String encodePassword(String password) {
return Base64.getEncoder().encodeToString(password.getBytes());
}
@ -1113,4 +1189,23 @@ public class UserResourceTest extends EntityResourceTest<User, CreateUser> {
.withAuthType(AuthenticationMechanism.AuthType.JWT)
.withConfig(new JWTAuthMechanism().withJWTTokenExpiry(JWTTokenExpiry.Unlimited)));
}
private CreateUser createBotUserRequest(TestInfo test, int index) {
return createBotUserRequest(getEntityName(test, index));
}
@Override
public CsvImportResult importCsv(String teamName, String csv, boolean dryRun) throws HttpResponseException {
WebTarget target = getCollection().path("/import");
target = target.queryParam("team", teamName);
target = !dryRun ? target.queryParam("dryRun", false) : target;
return TestUtils.putCsv(target, csv, CsvImportResult.class, Status.OK, ADMIN_AUTH_HEADERS);
}
@Override
protected String exportCsv(String teamName) throws HttpResponseException {
WebTarget target = getCollection().path("/export");
target = target.queryParam("team", teamName);
return TestUtils.get(target, String.class, ADMIN_AUTH_HEADERS);
}
}

5
openmetadata-spec/src/main/resources/json/schema/type/basic.json
View File

@ -85,6 +85,11 @@
"type": "string",
"format": "time"
},
"timezone": {
"description": "Timezone of the user in the format `America/Los_Angeles`, `Brazil/East`, etc.",
"type": "string",
"format": "timezone"
},
"entityLink": {
"description": "Link to an entity or field within an entity using this format `<#E::{entities}::{entityType}::{field}::{arrayFieldName}::{arrayFieldValue}`.",
"type": "string",