yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-10 14:57:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make LoadingCache members final variables (#12648)

Browse Source 
* Make LoadingCache members final variables

* Simplify singleton classes

* Reuse Entity fields instead of declaring them again
This commit is contained in:
Suresh Srinivas 2023-07-29 11:42:28 -07:00 committed by GitHub
parent 98b38e4e4d
commit b7e3242911
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
56 changed files with 387 additions and 645 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
openmetadata-service/src/main/java/org/openmetadata/service/Entity.java
View File

@ -261,11 +261,6 @@ public final class Entity {
return !ACTIVITY_FEED_EXCLUDED_ENTITIES.contains(entityType);
}
public static Fields getFields(String entityType, String fields) {
EntityRepository<?> entityRepository = Entity.getEntityRepository(entityType);
return entityRepository.getFields(fields);
}
public static Fields getFields(String entityType, List<String> fields) {
EntityRepository<?> entityRepository = Entity.getEntityRepository(entityType);
return entityRepository.getFields(String.join(",", fields));

36
openmetadata-service/src/main/java/org/openmetadata/service/events/scheduled/DataInsightsReportJob.java
View File

@ -108,7 +108,7 @@ public class DataInsightsReportJob implements Job {
if (!CommonUtil.nullOrEmpty(email)) {
emails.add(email);
} else {
team.getUsers().forEach(user -> emails.add(SubjectCache.getInstance().getUserById(user.getId()).getEmail()));
team.getUsers().forEach(user -> emails.add(SubjectCache.getUserById(user.getId()).getEmail()));
}
try {
DataInsightTotalAssetTemplate totalAssetTemplate =
@ -119,15 +119,14 @@ public class DataInsightsReportJob implements Job {
createOwnershipTemplate(searchClient, team.getName(), scheduleTime, currentTime, numberOfDaysChange);
DataInsightDescriptionAndOwnerTemplate tierTemplate =
createTierTemplate(searchClient, team.getName(), scheduleTime, currentTime, numberOfDaysChange);
EmailUtil.getInstance()
.sendDataInsightEmailNotificationToUser(
emails,
totalAssetTemplate,
descriptionTemplate,
ownershipTemplate,
tierTemplate,
EmailUtil.getInstance().getDataInsightReportSubject(),
EmailUtil.DATA_INSIGHT_REPORT_TEMPLATE);
EmailUtil.sendDataInsightEmailNotificationToUser(
emails,
totalAssetTemplate,
descriptionTemplate,
ownershipTemplate,
tierTemplate,
EmailUtil.getDataInsightReportSubject(),
EmailUtil.DATA_INSIGHT_REPORT_TEMPLATE);
} catch (Exception ex) {
LOG.error("[DataInsightReport] Failed for Team: {}, Reason : {}", team.getName(), ex.getMessage());
}
@ -149,15 +148,14 @@ public class DataInsightsReportJob implements Job {
createOwnershipTemplate(searchClient, null, scheduleTime, currentTime, numberOfDaysChange);
DataInsightDescriptionAndOwnerTemplate tierTemplate =
createTierTemplate(searchClient, null, scheduleTime, currentTime, numberOfDaysChange);
EmailUtil.getInstance()
.sendDataInsightEmailNotificationToUser(
emailList,
totalAssetTemplate,
descriptionTemplate,
ownershipTemplate,
tierTemplate,
EmailUtil.getInstance().getDataInsightReportSubject(),
EmailUtil.DATA_INSIGHT_REPORT_TEMPLATE);
EmailUtil.sendDataInsightEmailNotificationToUser(
emailList,
totalAssetTemplate,
descriptionTemplate,
ownershipTemplate,
tierTemplate,
EmailUtil.getDataInsightReportSubject(),
EmailUtil.DATA_INSIGHT_REPORT_TEMPLATE);
} catch (Exception ex) {
LOG.error("[DataInsightReport] Failed for Admin, Reason : {}", ex.getMessage(), ex);
}

29
openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/ActivityFeedAlertCache.java
View File

@ -10,36 +10,26 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.CheckForNull;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.entity.events.EventSubscription;
import org.openmetadata.schema.type.Include;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.EventSubscriptionRepository;
@Slf4j
public class ActivityFeedAlertCache {
private static final ActivityFeedAlertCache INSTANCE = new ActivityFeedAlertCache();
private static volatile boolean initialized = false;
protected static final LoadingCache<String, EventSubscription> eventSubCache =
protected static final LoadingCache<String, EventSubscription> EVENT_SUB_CACHE =
CacheBuilder.newBuilder()
.maximumSize(1000)
.expireAfterWrite(3, TimeUnit.MINUTES)
.build(new ActivityFeedAlertLoader());
protected static EventSubscriptionRepository eventSubscriptionRepository;
private static String activityFeedAlertName;
private static final String ACTIVITY_FEED_ALERT = "ActivityFeedAlert";
public static void initialize(String alertName, EventSubscriptionRepository repo) {
if (!initialized) {
eventSubscriptionRepository = repo;
initialized = true;
activityFeedAlertName = alertName;
}
private ActivityFeedAlertCache() {
// Private constructor for static class
}
public static ActivityFeedAlertCache getInstance() {
return INSTANCE;
}
public EventSubscription getActivityFeedAlert() throws EntityNotFoundException {
public static EventSubscription getActivityFeedAlert() throws EntityNotFoundException {
try {
return eventSubCache.get(activityFeedAlertName);
return EVENT_SUB_CACHE.get(ACTIVITY_FEED_ALERT);
} catch (ExecutionException | UncheckedExecutionException ex) {
throw new EntityNotFoundException(ex.getMessage());
}
@ -48,8 +38,7 @@ public class ActivityFeedAlertCache {
static class ActivityFeedAlertLoader extends CacheLoader<String, EventSubscription> {
@Override
public EventSubscription load(@CheckForNull String alertName) throws IOException {
EventSubscription alert =
eventSubscriptionRepository.getByName(null, alertName, eventSubscriptionRepository.getFields("*"));
EventSubscription alert = Entity.getEntityByName(Entity.EVENT_SUBSCRIPTION, alertName, "*", Include.NON_DELETED);
LOG.debug("Loaded Alert {}", alert);
return alert;
}

2
openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertUtil.java
View File

@ -213,7 +213,7 @@ public final class AlertUtil {
public static boolean shouldProcessActivityFeedRequest(ChangeEvent event) {
// Check Trigger Conditions
FilteringRules filteringRules = ActivityFeedAlertCache.getInstance().getActivityFeedAlert().getFilteringRules();
FilteringRules filteringRules = ActivityFeedAlertCache.getActivityFeedAlert().getFilteringRules();
return AlertUtil.shouldTriggerAlert(event.getEntityType(), filteringRules)
&& AlertUtil.evaluateAlertConditions(event, filteringRules.getRules());
}

4
openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluator.java
View File

@ -70,14 +70,14 @@ public class AlertsRuleEvaluator {
EntityReference ownerReference = entity.getOwner();
if (ownerReference != null) {
if (USER.equals(ownerReference.getType())) {
User user = SubjectCache.getInstance().getSubjectContext(ownerReference.getId()).getUser();
User user = SubjectCache.getSubjectContext(ownerReference.getId()).getUser();
for (String name : ownerNameList) {
if (user.getName().equals(name)) {
return true;
}
}
} else if (TEAM.equals(ownerReference.getType())) {
Team team = SubjectCache.getInstance().getTeam(ownerReference.getId());
Team team = SubjectCache.getTeam(ownerReference.getId());
for (String name : ownerNameList) {
if (team.getName().equals(name)) {
return true;

6
openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/SubscriptionPublisher.java
View File

@ -28,8 +28,6 @@ import org.openmetadata.schema.entity.events.EventSubscription;
import org.openmetadata.schema.entity.events.SubscriptionStatus;
import org.openmetadata.service.events.EventPubSub;
import org.openmetadata.service.events.errors.EventPublisherException;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.EventSubscriptionRepository;
import org.openmetadata.service.resources.events.EventResource;
/**
@ -52,11 +50,9 @@ import org.openmetadata.service.resources.events.EventResource;
public class SubscriptionPublisher extends AbstractAlertPublisher {
private final CountDownLatch shutdownLatch = new CountDownLatch(1);
@Getter private BatchEventProcessor<EventPubSub.ChangeEventHolder> processor;
private final EventSubscriptionRepository eventSubscriptionRepository;
public SubscriptionPublisher(EventSubscription eventSub, CollectionDAO dao) {
public SubscriptionPublisher(EventSubscription eventSub) {
super(eventSub);
this.eventSubscriptionRepository = new EventSubscriptionRepository(dao);
}
@SneakyThrows

2
openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/email/EmailPublisher.java
View File

@ -43,7 +43,7 @@ public class EmailPublisher extends SubscriptionPublisher {
private final CollectionDAO daoCollection;
public EmailPublisher(EventSubscription eventSub, CollectionDAO dao) {
super(eventSub, dao);
super(eventSub);
if (eventSub.getSubscriptionType() == EMAIL) {
this.emailAlertConfig = JsonUtils.convertValue(eventSub.getSubscriptionConfig(), EmailAlertConfig.class);
this.daoCollection = dao;

2
openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/gchat/GChatPublisher.java
View File

@ -45,7 +45,7 @@ public class GChatPublisher extends SubscriptionPublisher {
private final CollectionDAO daoCollection;
public GChatPublisher(EventSubscription eventSub, CollectionDAO dao) {
super(eventSub, dao);
super(eventSub);
if (eventSub.getSubscriptionType() == G_CHAT_WEBHOOK) {
this.daoCollection = dao;
this.webhook = JsonUtils.convertValue(eventSub.getSubscriptionConfig(), Webhook.class);

2
openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/generic/GenericPublisher.java
View File

@ -43,7 +43,7 @@ public class GenericPublisher extends SubscriptionPublisher {
private final CollectionDAO daoCollection;
public GenericPublisher(EventSubscription eventSub, CollectionDAO dao) {
super(eventSub, dao);
super(eventSub);
if (eventSub.getSubscriptionType() == GENERIC_WEBHOOK) {
this.daoCollection = dao;
this.webhook = JsonUtils.convertValue(eventSub.getSubscriptionConfig(), Webhook.class);

2
openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/msteams/MSTeamsPublisher.java
View File

@ -45,7 +45,7 @@ public class MSTeamsPublisher extends SubscriptionPublisher {
private final CollectionDAO daoCollection;
public MSTeamsPublisher(EventSubscription eventSub, CollectionDAO dao) {
super(eventSub, dao);
super(eventSub);
if (eventSub.getSubscriptionType() == MS_TEAMS_WEBHOOK) {
this.daoCollection = dao;
this.webhook = JsonUtils.convertValue(eventSub.getSubscriptionConfig(), Webhook.class);

2
openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/slack/SlackEventPublisher.java
View File

@ -43,7 +43,7 @@ public class SlackEventPublisher extends SubscriptionPublisher {
private final CollectionDAO daoCollection;
public SlackEventPublisher(EventSubscription eventSub, CollectionDAO dao) {
super(eventSub, dao);
super(eventSub);
if (eventSub.getSubscriptionType() == SLACK_WEBHOOK) {
this.daoCollection = dao;
this.webhook = JsonUtils.convertValue(eventSub.getSubscriptionConfig(), Webhook.class);

2
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -2068,7 +2068,7 @@ public interface CollectionDAO {
default List<TagLabel> getTags(String targetFQN) {
List<TagLabel> tags = getTagsInternal(FullyQualifiedName.buildHash(targetFQN));
tags.forEach(tagLabel -> tagLabel.setDescription(TagLabelCache.getInstance().getDescription(tagLabel)));
tags.forEach(tagLabel -> tagLabel.setDescription(TagLabelCache.getDescription(tagLabel)));
return tags;
}

4
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
View File

@ -1154,7 +1154,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
// When two tags have the same parent that is mutuallyExclusive, then throw an error
String parentFqn = FullyQualifiedName.getParentFQN(tagLabel.getTagFQN());
TagLabel stored = map.put(parentFqn, tagLabel);
if (stored != null && TagLabelCache.getInstance().mutuallyExclusive(tagLabel)) {
if (stored != null && TagLabelCache.mutuallyExclusive(tagLabel)) {
throw new IllegalArgumentException(CatalogExceptionMessage.mutuallyExclusiveLabels(tagLabel, stored));
}
}
@ -1598,7 +1598,7 @@ public abstract class EntityRepository<T extends EntityInterface> {
this.updatingUser =
updated.getUpdatedBy().equalsIgnoreCase(ADMIN_USER_NAME)
? new User().withName(ADMIN_USER_NAME).withIsAdmin(true)
: SubjectCache.getInstance().getSubjectContext(updated.getUpdatedBy()).getUser();
: SubjectCache.getSubjectContext(updated.getUpdatedBy()).getUser();
}
/** Compare original and updated entities and perform updates. Update the entity version and track changes. */

25
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java
View File

@ -126,10 +126,7 @@ public class FeedRepository {
// Validate about data entity is valid and get the owner for that entity
EntityLink about = EntityLink.parse(thread.getAbout());
EntityRepository<?> repository = Entity.getEntityRepository(about.getEntityType());
String field = "owner";
if (!repository.supportsOwner) {
field = "id";
}
String field = repository.supportsOwner ? "owner" : "";
EntityInterface aboutEntity = Entity.getEntity(about, field, ALL);
thread.withEntityId(aboutEntity.getId()); // Add entity id to thread
return createThread(thread, about, aboutEntity.getOwner());
@ -139,7 +136,7 @@ public class FeedRepository {
private Thread createThread(Thread thread, EntityLink about, EntityReference entityOwner)
throws JsonProcessingException {
// Validate user creating thread
User createdByUser = SubjectCache.getInstance().getUser(thread.getCreatedBy());
User createdByUser = SubjectCache.getUser(thread.getCreatedBy());
if (thread.getType() == ThreadType.Task) {
thread.withTask(thread.getTask().withId(getNextTaskId())); // Assign taskId for a task
@ -326,7 +323,7 @@ public class FeedRepository {
@Transaction
public Thread addPostToThread(String id, Post post, String userName) throws IOException {
// Validate the user posting the message
User fromUser = SubjectCache.getInstance().getUser(post.getFrom());
User fromUser = SubjectCache.getUser(post.getFrom());
// Update the thread with the new post
Thread thread = EntityUtil.validate(id, dao.feedDAO().findById(id), Thread.class);
@ -387,10 +384,6 @@ public class FeedRepository {
return new DeleteResponse<>(thread, RestUtil.ENTITY_DELETED);
}
public EntityReference getOwnerReference(String username) {
return dao.userDAO().findEntityByName(EntityInterfaceUtil.quoteName(username)).getEntityReference();
}
@Transaction
public ThreadCount getThreadsCount(FeedFilter filter, String link) {
List<List<String>> result;
@ -475,7 +468,7 @@ public class FeedRepository {
total = filteredThreads.getTotalCount();
} else {
// Only data assets are added as about
User user = userId != null ? SubjectCache.getInstance().getUserById(userId) : null;
User user = userId != null ? SubjectCache.getUserById(userId) : null;
List<String> teamNameHash = getTeamNames(user);
String userNameHash = getUserNameHash(user);
List<String> jsons =
@ -631,7 +624,7 @@ public class FeedRepository {
public void checkPermissionsForResolveTask(Thread thread, boolean closeTask, SecurityContext securityContext)
throws IOException {
String userName = securityContext.getUserPrincipal().getName();
User user = SubjectCache.getInstance().getUser(userName);
User user = SubjectCache.getUser(userName);
EntityLink about = EntityLink.parse(thread.getAbout());
EntityReference aboutRef = EntityUtil.validateEntityLink(about);
if (Boolean.TRUE.equals(user.getIsAdmin())) {
@ -830,7 +823,7 @@ public class FeedRepository {
/** Return the tasks created by or assigned to the user. */
private FilteredThreads getTasksOfUser(FeedFilter filter, String userId, int limit) throws IOException {
String username = SubjectCache.getInstance().getUserById(userId).getName();
String username = SubjectCache.getUserById(userId).getName();
List<String> teamIds = getTeamIds(userId);
List<String> userTeamJsonPostgres = getUserTeamJsonPostgres(userId, teamIds);
String userTeamJsonMysql = getUserTeamJsonMysql(userId, teamIds);
@ -845,7 +838,7 @@ public class FeedRepository {
/** Return the tasks created by the user. */
private FilteredThreads getTasksAssignedBy(FeedFilter filter, String userId, int limit) throws IOException {
String username = SubjectCache.getInstance().getUserById(userId).getName();
String username = SubjectCache.getUserById(userId).getName();
List<String> jsons = dao.feedDAO().listTasksAssigned(username, limit, filter.getCondition());
List<Thread> threads = JsonUtils.readObjects(jsons, Thread.class);
int totalCount = dao.feedDAO().listCountTasksAssignedBy(username, filter.getCondition(false));
@ -869,7 +862,7 @@ public class FeedRepository {
/** Returns the threads where the user or the team they belong to were mentioned by other users with @mention. */
private FilteredThreads getThreadsByMentions(FeedFilter filter, String userId, int limit) throws IOException {
User user = SubjectCache.getInstance().getUserById(userId);
User user = SubjectCache.getUserById(userId);
String userNameHash = getUserNameHash(user);
// Return the threads where the user or team was mentioned
List<String> teamNamesHash = getTeamNames(user);
@ -891,7 +884,7 @@ public class FeedRepository {
private List<String> getTeamIds(String userId) {
List<String> teamIds = null;
if (userId != null) {
User user = SubjectCache.getInstance().getUserById(userId);
User user = SubjectCache.getUserById(userId);
teamIds = listOrEmpty(user.getTeams()).stream().map(ref -> ref.getId().toString()).collect(Collectors.toList());
}
return nullOrEmpty(teamIds) ? List.of(StringUtils.EMPTY) : teamIds;

4
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PolicyRepository.java
View File

@ -77,7 +77,7 @@ public class PolicyRepository extends EntityRepository<Policy> {
public void storeEntity(Policy policy, boolean update) throws IOException {
store(policy, update);
if (update) {
PolicyCache.getInstance().invalidatePolicy(policy.getId());
PolicyCache.invalidatePolicy(policy.getId());
}
}
@ -102,7 +102,7 @@ public class PolicyRepository extends EntityRepository<Policy> {
@Override
protected void cleanup(Policy policy) throws IOException {
super.cleanup(policy);
PolicyCache.getInstance().invalidatePolicy(policy.getId());
PolicyCache.invalidatePolicy(policy.getId());
}
public void validateRules(Policy policy) {

4
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/RoleRepository.java
View File

@ -92,7 +92,7 @@ public class RoleRepository extends EntityRepository<Role> {
role.withPolicies(null);
store(role, update);
if (update) {
RoleCache.getInstance().invalidateRole(role.getId());
RoleCache.invalidateRole(role.getId());
}
role.withPolicies(policies);
}
@ -120,7 +120,7 @@ public class RoleRepository extends EntityRepository<Role> {
@Override
protected void cleanup(Role role) throws IOException {
super.cleanup(role);
RoleCache.getInstance().invalidateRole(role.getId());
RoleCache.invalidateRole(role.getId());
}
/** Handles entity updated from PUT and POST operation. */

2
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java
View File

@ -154,7 +154,7 @@ public class SystemRepository {
}
dao.insertSettings(setting.getConfigType().toString(), JsonUtils.pojoToJson(setting.getConfigValue()));
// Invalidate Cache
SettingsCache.getInstance().invalidateSettings(setting.getConfigType().value());
SettingsCache.invalidateSettings(setting.getConfigType().value());
} catch (Exception ex) {
LOG.error("Failing in Updating Setting.", ex);
throw new CustomExceptionMessage(Response.Status.INTERNAL_SERVER_ERROR, ex.getMessage());

8
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java
View File

@ -144,7 +144,7 @@ public class TeamRepository extends EntityRepository<Team> {
store(team, update);
if (update) {
SubjectCache.getInstance().invalidateTeam(team.getId());
SubjectCache.invalidateTeam(team.getId());
}
// Restore the relationships
@ -216,7 +216,7 @@ public class TeamRepository extends EntityRepository<Team> {
}
}
super.cleanup(team);
SubjectCache.getInstance().invalidateTeam(team.getId());
SubjectCache.invalidateTeam(team.getId());
}
@Override
@ -233,7 +233,7 @@ public class TeamRepository extends EntityRepository<Team> {
}
private List<EntityReference> getInheritedRoles(Team team) throws IOException {
return SubjectCache.getInstance().getRolesForTeams(getParentsForInheritedRoles(team));
return SubjectCache.getRolesForTeams(getParentsForInheritedRoles(team));
}
private TeamHierarchy getTeamHierarchy(Team team) {
@ -619,7 +619,7 @@ public class TeamRepository extends EntityRepository<Team> {
continue; // Parent is being created by CSV import
}
// Else the parent should already exist
if (!SubjectCache.getInstance().isInTeam(team.getName(), parentRef)) {
if (!SubjectCache.isInTeam(team.getName(), parentRef)) {
importFailure(
printer, invalidTeam(4, team.getName(), importedTeam.getName(), parentRef.getName()), csvRecord);
processRecord = false;

10
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java
View File

@ -121,7 +121,7 @@ public class UserRepository extends EntityRepository<User> {
return null; // No inherited roles for bots
}
getTeams(user);
return SubjectCache.getInstance() != null ? SubjectCache.getInstance().getRolesForTeams(getTeams(user)) : null;
return SubjectCache.getRolesForTeams(getTeams(user));
}
@Override
@ -140,7 +140,7 @@ public class UserRepository extends EntityRepository<User> {
store(user, update);
if (update) {
SubjectCache.getInstance().invalidateUser(user.getName());
SubjectCache.invalidateUser(user.getName());
}
// Restore the relationships
@ -175,13 +175,13 @@ public class UserRepository extends EntityRepository<User> {
@Override
protected void postDelete(User entity) {
SubjectCache.getInstance().invalidateUser(entity.getName());
SubjectCache.invalidateUser(entity.getName());
}
@Override
protected void cleanup(User user) throws IOException {
super.cleanup(user);
SubjectCache.getInstance().invalidateUser(user.getName());
SubjectCache.invalidateUser(user.getName());
}
@Override
@ -436,7 +436,7 @@ public class UserRepository extends EntityRepository<User> {
continue; // Team is same as the team to which CSV is being imported, then it is in the same hierarchy
}
// Else the parent should already exist
if (!SubjectCache.getInstance().isInTeam(team.getName(), teamRef)) {
if (!SubjectCache.isInTeam(team.getName(), teamRef)) {
importFailure(printer, invalidTeam(6, team.getName(), user, teamRef.getName()), csvRecord);
processRecord = false;
}

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotResource.java
View File

@ -103,7 +103,7 @@ public class BotResource extends EntityResource<Bot, BotRepository> {
// Add role corresponding to the bot to the user
// we need to set a mutable list here
user.setRoles(getRoleForBot(bot.getName()));
user = UserUtil.addOrUpdateBotUser(user, config);
user = UserUtil.addOrUpdateBotUser(user);
bot.withBotUser(user.getEntityReference());
repository.initializeEntity(bot);
}

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java
View File

@ -63,7 +63,6 @@ import org.openmetadata.schema.type.SubscriptionResourceDescriptor;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.events.scheduled.ReportsHandler;
import org.openmetadata.service.events.subscription.ActivityFeedAlertCache;
import org.openmetadata.service.events.subscription.AlertUtil;
import org.openmetadata.service.events.subscription.EventsSubscriptionRegistry;
import org.openmetadata.service.exception.EntityNotFoundException;
@ -125,7 +124,6 @@ public class EventSubscriptionResource extends EntityResource<EventSubscription,
try {
repository.initSeedDataFromResources();
EventsSubscriptionRegistry.initialize(listOrEmpty(EventSubscriptionResource.getDescriptors()));
ActivityFeedAlertCache.initialize("ActivityFeedAlert", repository);
searchClient = IndexUtil.getSearchClient(config.getElasticSearchConfiguration(), daoCollection);
ReportsHandler.initialize(daoCollection, searchClient);
initializeEventSubscriptions();

7
openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java
View File

@ -56,14 +56,12 @@ import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.MetadataOperation;
import org.openmetadata.schema.utils.EntityInterfaceUtil;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.exception.CatalogExceptionMessage;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.GlossaryTermRepository;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.resources.tags.TagLabelCache;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.RestUtil;
@ -91,11 +89,6 @@ public class GlossaryTermResource extends EntityResource<GlossaryTerm, GlossaryT
return term;
}
@Override
public void initialize(OpenMetadataApplicationConfig config) {
TagLabelCache.initialize();
}
public GlossaryTermResource(CollectionDAO dao, Authorizer authorizer) {
super(GlossaryTerm.class, new GlossaryTermRepository(dao), authorizer);
}

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java
View File

@ -67,7 +67,6 @@ import org.openmetadata.service.resources.CollectionRegistry;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.policyevaluator.CompiledRule;
import org.openmetadata.service.security.policyevaluator.PolicyCache;
import org.openmetadata.service.security.policyevaluator.RuleEvaluator;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ResultList;
@ -106,7 +105,6 @@ public class PolicyResource extends EntityResource<Policy, PolicyRepository> {
public void initialize(OpenMetadataApplicationConfig config) throws IOException {
// Load any existing rules from database, before loading seed data.
repository.initSeedDataFromResources();
PolicyCache.initialize();
}
@Override

33
openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java
View File

@ -34,16 +34,18 @@ import org.openmetadata.service.util.JsonUtils;
@Slf4j
public class SettingsCache {
private static final SettingsCache INSTANCE = new SettingsCache();
private static volatile boolean initialized = false;
protected static LoadingCache<String, Settings> cache;
protected static final LoadingCache<String, Settings> CACHE =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new SettingsLoader());
protected static SystemRepository systemRepository;
private SettingsCache() {
// Private constructor for singleton
}
// Expected to be called only once from the DefaultAuthorizer
public static void initialize(CollectionDAO dao, OpenMetadataApplicationConfig config) {
if (!initialized) {
cache =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new SettingsLoader());
systemRepository = new SystemRepository(dao.systemDAO());
initialized = true;
createDefaultConfiguration(config);
@ -72,13 +74,9 @@ public class SettingsCache {
}
}
public static SettingsCache getInstance() {
return INSTANCE;
}
public <T> T getSetting(SettingsType settingName, Class<T> clazz) {
public static <T> T getSetting(SettingsType settingName, Class<T> clazz) {
try {
String json = JsonUtils.pojoToJson(cache.get(settingName.toString()).getConfigValue());
String json = JsonUtils.pojoToJson(CACHE.get(settingName.toString()).getConfigValue());
return JsonUtils.readValue(json, clazz);
} catch (Exception ex) {
LOG.error("Failed to fetch Settings . Setting {}", settingName, ex);
@ -86,23 +84,14 @@ public class SettingsCache {
}
}
public Settings getSetting(SettingsType settingName) {
try {
return cache.get(settingName.toString());
} catch (Exception ex) {
LOG.error("Failed to fetch Settings . Setting {}", settingName, ex);
throw new EntityNotFoundException("Setting not found");
}
}
public static void cleanUp() {
cache.invalidateAll();
CACHE.invalidateAll();
initialized = false;
}
public void invalidateSettings(String settingsName) {
public static void invalidateSettings(String settingsName) {
try {
cache.invalidate(settingsName);
CACHE.invalidate(settingsName);
} catch (Exception ex) {
LOG.error("Failed to invalidate cache for settings {}", settingsName, ex);
}

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/system/ConfigResource.java
View File

@ -102,7 +102,7 @@ public class ConfigResource {
schema = @Schema(implementation = AuthenticationConfiguration.class)))
})
public LogoConfiguration getCustomLogoConfig() {
return SettingsCache.getInstance().getSetting(SettingsType.CUSTOM_LOGO_CONFIGURATION, LogoConfiguration.class);
return SettingsCache.getSetting(SettingsType.CUSTOM_LOGO_CONFIGURATION, LogoConfiguration.class);
}
@GET

98
openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelCache.java
View File

@ -13,6 +13,8 @@
package org.openmetadata.service.resources.tags;
import static org.openmetadata.schema.type.Include.NON_DELETED;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@ -31,11 +33,6 @@ import org.openmetadata.schema.type.TagLabel.TagSource;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.CatalogExceptionMessage;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.ClassificationRepository;
import org.openmetadata.service.jdbi3.GlossaryRepository;
import org.openmetadata.service.jdbi3.GlossaryTermRepository;
import org.openmetadata.service.jdbi3.TagRepository;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.FullyQualifiedName;
/**
@ -43,96 +40,67 @@ import org.openmetadata.service.util.FullyQualifiedName;
*/
@Slf4j
public class TagLabelCache {
private static final TagLabelCache INSTANCE = new TagLabelCache();
private static volatile boolean initialized = false;
// Tag fqn to Tag
protected static final LoadingCache<String, Tag> TAG_CACHE =
CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(2, TimeUnit.MINUTES).build(new TagLoader());
// Classification name to Classification
protected static final LoadingCache<String, Classification> CLASSIFICATION_CACHE =
CacheBuilder.newBuilder().maximumSize(25).expireAfterWrite(2, TimeUnit.MINUTES).build(new ClassificationLoader());
protected static TagRepository tagRepository;
protected static ClassificationRepository tagClassificationRepository;
protected static LoadingCache<String, Tag> tagCache; // Tag fqn to Tag
protected static LoadingCache<String, Classification> classificationCache; // Classification name to Classification
protected static GlossaryTermRepository glossaryTermRepository;
protected static GlossaryRepository glossaryRepository;
// Glossary term fqn to GlossaryTerm
protected static LoadingCache<String, GlossaryTerm> glossaryTermCache;
protected static final LoadingCache<String, GlossaryTerm> GLOSSARY_TERM_CACHE =
CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(2, TimeUnit.MINUTES).build(new GlossaryTermLoader());
protected static LoadingCache<String, Glossary> glossaryCache; // Glossary fqn to Glossary
// Glossary fqn to Glossary
protected static final LoadingCache<String, Glossary> GLOSSARY_CACHE =
CacheBuilder.newBuilder().maximumSize(25).expireAfterWrite(2, TimeUnit.MINUTES).build(new GlossaryLoader());
// Expected to be called only once from the TagResource during initialization
public static void initialize() {
if (!initialized) {
classificationCache =
CacheBuilder.newBuilder()
.maximumSize(25)
.expireAfterWrite(2, TimeUnit.MINUTES)
.build(new ClassificationLoader());
tagCache =
CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(2, TimeUnit.MINUTES).build(new TagLoader());
tagRepository = (TagRepository) Entity.getEntityRepository(Entity.TAG);
tagClassificationRepository = (ClassificationRepository) Entity.getEntityRepository(Entity.CLASSIFICATION);
glossaryCache =
CacheBuilder.newBuilder().maximumSize(25).expireAfterWrite(2, TimeUnit.MINUTES).build(new GlossaryLoader());
glossaryTermCache =
CacheBuilder.newBuilder()
.maximumSize(100)
.expireAfterWrite(2, TimeUnit.MINUTES)
.build(new GlossaryTermLoader());
glossaryTermRepository = (GlossaryTermRepository) Entity.getEntityRepository(Entity.GLOSSARY_TERM);
glossaryRepository = (GlossaryRepository) Entity.getEntityRepository(Entity.GLOSSARY);
initialized = true;
} else {
LOG.info("Subject cache is already initialized");
}
}
public static TagLabelCache getInstance() {
return INSTANCE;
private TagLabelCache() {
// Private constructor for utility class
}
public static void cleanUp() {
classificationCache.cleanUp();
tagCache.cleanUp();
glossaryCache.cleanUp();
glossaryTermCache.cleanUp();
initialized = false;
CLASSIFICATION_CACHE.cleanUp();
TAG_CACHE.cleanUp();
GLOSSARY_CACHE.cleanUp();
GLOSSARY_TERM_CACHE.cleanUp();
}
public Classification getClassification(String classificationName) {
public static Classification getClassification(String classificationName) {
try {
return classificationCache.get(classificationName);
return CLASSIFICATION_CACHE.get(classificationName);
} catch (ExecutionException | UncheckedExecutionException ex) {
throw EntityNotFoundException.byMessage(
CatalogExceptionMessage.entityNotFound(Entity.CLASSIFICATION, classificationName));
}
}
public Tag getTag(String tagFqn) {
public static Tag getTag(String tagFqn) {
try {
return tagCache.get(tagFqn);
return TAG_CACHE.get(tagFqn);
} catch (ExecutionException | UncheckedExecutionException ex) {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Entity.TAG, tagFqn));
}
}
public Glossary getGlossary(String glossaryName) {
public static Glossary getGlossary(String glossaryName) {
try {
return glossaryCache.get(glossaryName);
return GLOSSARY_CACHE.get(glossaryName);
} catch (ExecutionException | UncheckedExecutionException ex) {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY, glossaryName));
}
}
public GlossaryTerm getGlossaryTerm(String glossaryTermFqn) {
public static GlossaryTerm getGlossaryTerm(String glossaryTermFqn) {
try {
return glossaryTermCache.get(glossaryTermFqn);
return GLOSSARY_TERM_CACHE.get(glossaryTermFqn);
} catch (ExecutionException | UncheckedExecutionException ex) {
throw EntityNotFoundException.byMessage(
CatalogExceptionMessage.entityNotFound(Entity.GLOSSARY_TERM, glossaryTermFqn));
}
}
public String getDescription(TagLabel label) {
public static String getDescription(TagLabel label) {
if (label.getSource() == TagSource.CLASSIFICATION) {
return getTag(label.getTagFQN()).getDescription();
} else if (label.getSource() == TagSource.GLOSSARY) {
@ -143,7 +111,7 @@ public class TagLabelCache {
}
/** Returns true if the parent of the tag label is mutually exclusive */
public boolean mutuallyExclusive(TagLabel label) {
public static boolean mutuallyExclusive(TagLabel label) {
String[] fqnParts = FullyQualifiedName.split(label.getTagFQN());
String parentFqn = FullyQualifiedName.getParentFQN(fqnParts);
boolean rootParent = fqnParts.length == 2;
@ -163,7 +131,7 @@ public class TagLabelCache {
static class TagLoader extends CacheLoader<String, Tag> {
@Override
public Tag load(@CheckForNull String tagName) throws IOException {
Tag tag = tagRepository.getByName(null, tagName, Fields.EMPTY_FIELDS);
Tag tag = Entity.getEntityByName(Entity.TAG, tagName, "", NON_DELETED);
LOG.info("Loaded tag {}:{}", tag.getName(), tag.getId());
return tag;
}
@ -173,7 +141,7 @@ public class TagLabelCache {
@Override
public Classification load(@CheckForNull String classificationName) throws IOException {
Classification classification =
tagClassificationRepository.getByName(null, classificationName, Fields.EMPTY_FIELDS);
Entity.getEntityByName(Entity.CLASSIFICATION, classificationName, "", NON_DELETED);
LOG.info("Loaded classification {}:{}", classification.getName(), classification.getId());
return classification;
}
@ -182,7 +150,7 @@ public class TagLabelCache {
static class GlossaryTermLoader extends CacheLoader<String, GlossaryTerm> {
@Override
public GlossaryTerm load(@CheckForNull String glossaryTermName) throws IOException {
GlossaryTerm glossaryTerm = glossaryTermRepository.getByName(null, glossaryTermName, Fields.EMPTY_FIELDS);
GlossaryTerm glossaryTerm = Entity.getEntityByName(Entity.GLOSSARY_TERM, glossaryTermName, "", NON_DELETED);
LOG.info("Loaded glossaryTerm {}:{}", glossaryTerm.getName(), glossaryTerm.getId());
return glossaryTerm;
}
@ -191,7 +159,7 @@ public class TagLabelCache {
static class GlossaryLoader extends CacheLoader<String, Glossary> {
@Override
public Glossary load(@CheckForNull String glossaryName) throws IOException {
Glossary glossary = glossaryRepository.getByName(null, glossaryName, Fields.EMPTY_FIELDS);
Glossary glossary = Entity.getEntityByName(Entity.GLOSSARY, glossaryName, "", NON_DELETED);
LOG.info("Loaded glossary {}:{}", glossary.getName(), glossary.getId());
return glossary;
}

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java
View File

@ -63,7 +63,6 @@ import org.openmetadata.service.jdbi3.RoleRepository;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.policyevaluator.RoleCache;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.RestUtil;
import org.openmetadata.service.util.ResultList;
@ -112,7 +111,6 @@ public class RoleResource extends EntityResource<Role, RoleRepository> {
}
repository.initializeEntity(role);
}
RoleCache.initialize();
}
public static class RoleList extends ResultList<Role> {

14
openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java
View File

@ -132,7 +132,6 @@ import org.openmetadata.service.security.auth.UserTokenCache;
import org.openmetadata.service.security.jwt.JWTTokenGenerator;
import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.security.policyevaluator.ResourceContext;
import org.openmetadata.service.security.policyevaluator.SubjectCache;
import org.openmetadata.service.security.saml.JwtTokenCacheManager;
import org.openmetadata.service.util.EmailUtil;
import org.openmetadata.service.util.EntityUtil;
@ -194,8 +193,6 @@ public class UserResource extends EntityResource<User, UserRepository> {
this.authenticationConfiguration = config.getAuthenticationConfiguration();
SmtpSettings smtpSettings = config.getSmtpSettings();
this.isEmailServiceEnabled = smtpSettings != null && smtpSettings.getEnableSmtpServer();
// Keep this before initializeUsers, else getUpdater() will fail
SubjectCache.initialize();
this.repository.initializeUsers(config);
}
@ -523,7 +520,7 @@ public class UserResource extends EntityResource<User, UserRepository> {
authHandler.sendInviteMailToUser(
uriInfo,
user,
String.format("Welcome to %s", EmailUtil.getInstance().getEmailingEntity()),
String.format("Welcome to %s", EmailUtil.getEmailingEntity()),
create.getCreatePasswordType(),
create.getPassword());
} catch (Exception ex) {
@ -635,7 +632,7 @@ public class UserResource extends EntityResource<User, UserRepository> {
RestUtil.PutResponse<User> response = repository.createOrUpdate(uriInfo, user);
addHref(uriInfo, response.getEntity());
// Invalidate Bot Token in Cache
BotTokenCache.getInstance().invalidateToken(user.getName());
BotTokenCache.invalidateToken(user.getName());
return response.toResponse();
}
@ -910,10 +907,7 @@ public class UserResource extends EntityResource<User, UserRepository> {
try {
// send a mail to the User with the Update
authHandler.sendPasswordResetLink(
uriInfo,
registeredUser,
EmailUtil.getInstance().getPasswordResetSubject(),
EmailUtil.PASSWORD_RESET_TEMPLATE_FILE);
uriInfo, registeredUser, EmailUtil.getPasswordResetSubject(), EmailUtil.PASSWORD_RESET_TEMPLATE_FILE);
} catch (Exception ex) {
LOG.error("Error in sending mail for reset password" + ex.getMessage());
return Response.status(424).entity(new ErrorMessage(424, EMAIL_SENDING_ISSUE)).build();
@ -1117,7 +1111,7 @@ public class UserResource extends EntityResource<User, UserRepository> {
List<String> ids = request.getTokenIds().stream().map(UUID::toString).collect(Collectors.toList());
tokenRepository.deleteAllToken(ids);
}
UserTokenCache.getInstance().invalidateToken(user.getName());
UserTokenCache.invalidateToken(user.getName());
List<TokenInterface> tokens =
tokenRepository.findByUserIdAndType(user.getId().toString(), TokenType.PERSONAL_ACCESS_TOKEN.value());
return Response.status(Response.Status.OK).entity(new ResultList<>(tokens)).build();

13
openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java
View File

@ -1,22 +1,10 @@
package org.openmetadata.service.search;
public class EntityBuilderConstant {
public static final String FIELD_OWNER = "owner";
public static final String FIELD_NAME = "name";
public static final String FIELD_DESCRIPTION = "description";
public static final String FIELD_FOLLOWERS = "followers";
public static final String FIELD_VOTES = "votes";
public static final String FIELD_TAGS = "tags";
public static final String FIELD_DELETED = "deleted";
public static final String FIELD_PIPELINE_STATUS = "pipelineStatus";
public static final String FIELD_DISPLAY_NAME = "displayName";
public static final String FIELD_EXTENSION = "extension";
public static final String FIELD_USAGE_SUMMARY = "usageSummary";
public static final String UNIFIED = "unified";
public static final String ES_MESSAGE_SCHEMA_FIELD = "messageSchema.schemaFields.name";
public static final String ES_TAG_FQN_FIELD = "tags.tagFQN";
public static final String NAME_KEYWORD = "name.keyword";
public static final String DISPLAY_NAME = "displayName";
public static final String FIELD_NAME_NGRAM = "name.ngram";
public static final String DISPLAY_NAME_KEYWORD = "displayName.keyword";
public static final String FIELD_DISPLAY_NAME_NGRAM = "displayName.ngram";
@ -25,7 +13,6 @@ public class EntityBuilderConstant {
public static final String POST_TAG = "</span>";
public static final Integer MAX_AGGREGATE_SIZE = 50;
public static final Integer MAX_RESULT_HITS = 10000;
public static final String DESCRIPTION = "description";
public static final String QUERY = "query";
public static final String QUERY_NGRAM = "query.ngram";

52
openmetadata-service/src/main/java/org/openmetadata/service/search/elasticSearch/ElasticSearchClientImpl.java
View File

@ -403,20 +403,20 @@ public class ElasticSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildPipelineSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 15.0f)
.field(FIELD_DISPLAY_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.DESCRIPTION, 1.0f)
.field(FIELD_DESCRIPTION, 1.0f)
.field("tasks.name", 2.0f)
.field("tasks.description", 1.0f)
.defaultOperator(Operator.AND)
.fuzziness(Fuzziness.AUTO);
HighlightBuilder.Field highlightPipelineName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME);
highlightPipelineName.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(EntityBuilderConstant.DESCRIPTION);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightTasks = new HighlightBuilder.Field("tasks.name");
highlightTasks.highlighterType(EntityBuilderConstant.UNIFIED);
@ -436,20 +436,20 @@ public class ElasticSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildMlModelSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 15.0f)
.field(Entity.FIELD_DISPLAY_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.DESCRIPTION, 1.0f)
.field(FIELD_DESCRIPTION, 1.0f)
.field("mlFeatures.name", 2.0f)
.field("mlFeatures.description", 1.0f)
.defaultOperator(Operator.AND)
.fuzziness(Fuzziness.AUTO);
HighlightBuilder.Field highlightPipelineName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME);
highlightPipelineName.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(EntityBuilderConstant.DESCRIPTION);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightTasks = new HighlightBuilder.Field("mlFeatures.name");
highlightTasks.highlighterType(EntityBuilderConstant.UNIFIED);
@ -467,14 +467,14 @@ public class ElasticSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildTopicSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 15.0f)
.field(Entity.FIELD_DISPLAY_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_NAME_NGRAM)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION, 1.0f)
.field(Entity.FIELD_DESCRIPTION, 1.0f)
.field(EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD, 2.0f)
.field("messageSchema.schemaFields.description", 1.0f)
.field("messageSchema.schemaFields.children.name", 2.0f)
@ -503,14 +503,14 @@ public class ElasticSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildDashboardSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 15.0f)
.field(Entity.FIELD_DISPLAY_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_NAME_NGRAM)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION, 1.0f)
.field(Entity.FIELD_DESCRIPTION, 1.0f)
.field("charts.name", 2.0f)
.field("charts.description", 1.0f)
.defaultOperator(Operator.AND)
@ -541,13 +541,13 @@ public class ElasticSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildTableSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryStringBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 15.0f)
.field(Entity.FIELD_DISPLAY_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_NAME_NGRAM)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION, 1.0f)
.field(Entity.FIELD_DESCRIPTION, 1.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.field("columns.name.keyword", 10.0f)
.field("columns.name", 2.0f)
@ -569,7 +569,7 @@ public class ElasticSearchClientImpl implements SearchClient {
queryBuilder.boostMode(CombineFunction.SUM);
HighlightBuilder.Field highlightTableName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME);
highlightTableName.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(EntityBuilderConstant.DESCRIPTION);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder hb = new HighlightBuilder();
HighlightBuilder.Field highlightColumns = new HighlightBuilder.Field("columns.name");
@ -597,7 +597,7 @@ public class ElasticSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildUserOrTeamSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 3.0f)
.field(Entity.FIELD_DISPLAY_NAME, 3.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 5.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 2.0f)
@ -610,16 +610,16 @@ public class ElasticSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildGlossaryTermSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 10.0f)
.field(Entity.FIELD_DISPLAY_NAME, 10.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM, 1.0f)
.field(FIELD_NAME, 10.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 10.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 10.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 10.0f)
.field(Entity.FIELD_DISPLAY_NAME, 10.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field("synonyms", 5.0f)
.field("synonyms.ngram")
.field(EntityBuilderConstant.DESCRIPTION, 3.0f)
.field(FIELD_DESCRIPTION, 3.0f)
.field("glossary.name", 5.0f)
.field("glossary.displayName", 5.0f)
.field("glossary.displayName.ngram")
@ -658,10 +658,10 @@ public class ElasticSearchClientImpl implements SearchClient {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(FIELD_NAME, 10.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 10.0f)
.field(Entity.FIELD_DISPLAY_NAME, 10.0f)
.field(EntityBuilderConstant.FIELD_NAME_NGRAM, 1.0f)
.field("classification.name", 1.0f)
.field(EntityBuilderConstant.DESCRIPTION, 3.0f)
.field(FIELD_DESCRIPTION, 3.0f)
.defaultOperator(Operator.AND)
.fuzziness(Fuzziness.AUTO);
@ -684,10 +684,10 @@ public class ElasticSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildContainerSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 15.0f)
.field(Entity.FIELD_DISPLAY_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION, 1.0f)
.field(Entity.FIELD_DESCRIPTION, 1.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 25.0f)
@ -702,7 +702,7 @@ public class ElasticSearchClientImpl implements SearchClient {
.fuzziness(Fuzziness.AUTO);
HighlightBuilder.Field highlightContainerName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME);
highlightContainerName.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(EntityBuilderConstant.DESCRIPTION);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder hb = new HighlightBuilder();
HighlightBuilder.Field highlightColumns = new HighlightBuilder.Field("dataModel.columns.name");
@ -728,16 +728,16 @@ public class ElasticSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildQuerySearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.DISPLAY_NAME, 10.0f)
.field(FIELD_DISPLAY_NAME, 10.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(EntityBuilderConstant.QUERY, 10.0f)
.field(EntityBuilderConstant.QUERY_NGRAM)
.field(EntityBuilderConstant.DESCRIPTION, 1.0f)
.field(FIELD_DESCRIPTION, 1.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.defaultOperator(Operator.AND)
.fuzziness(Fuzziness.AUTO);
HighlightBuilder.Field highlightGlossaryName = new HighlightBuilder.Field(EntityBuilderConstant.DISPLAY_NAME);
HighlightBuilder.Field highlightGlossaryName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME);
highlightGlossaryName.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(EntityBuilderConstant.UNIFIED);
@ -756,7 +756,7 @@ public class ElasticSearchClientImpl implements SearchClient {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(FIELD_NAME, 10.0f)
.field(EntityBuilderConstant.DESCRIPTION, 3.0f)
.field(FIELD_DESCRIPTION, 3.0f)
.field("testSuite.fullyQualifiedName", 10.0f)
.field("testSuite.name", 10.0f)
.field("testSuite.description", 3.0f)

70
openmetadata-service/src/main/java/org/openmetadata/service/search/openSearch/OpenSearchClientImpl.java
View File

@ -154,11 +154,11 @@ public class OpenSearchClientImpl implements SearchClient {
this.dao = dao;
}
private static NamedXContentRegistry xContentRegistry;
private static final NamedXContentRegistry X_CONTENT_REGISTRY;
static {
SearchModule searchModule = new SearchModule(Settings.EMPTY, false, List.of());
xContentRegistry = new NamedXContentRegistry(searchModule.getNamedXContents());
X_CONTENT_REGISTRY = new NamedXContentRegistry(searchModule.getNamedXContents());
}
@Override
@ -286,7 +286,7 @@ public class OpenSearchClientImpl implements SearchClient {
XContentParser filterParser =
XContentType.JSON
.xContent()
.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, request.getQueryFilter());
.createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, request.getQueryFilter());
QueryBuilder filter = SearchSourceBuilder.fromXContent(filterParser).query();
BoolQueryBuilder newQuery = QueryBuilders.boolQuery().must(searchSourceBuilder.query()).filter(filter);
searchSourceBuilder.query(newQuery);
@ -300,7 +300,7 @@ public class OpenSearchClientImpl implements SearchClient {
XContentParser filterParser =
XContentType.JSON
.xContent()
.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, request.getPostFilter());
.createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, request.getPostFilter());
QueryBuilder filter = SearchSourceBuilder.fromXContent(filterParser).query();
searchSourceBuilder.postFilter(filter);
} catch (Exception ex) {
@ -344,8 +344,8 @@ public class OpenSearchClientImpl implements SearchClient {
public Response aggregate(String index, String fieldName, String value, String query) throws IOException {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
XContentParser filterParser =
XContentType.JSON.xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, query);
QueryBuilder filter = searchSourceBuilder.fromXContent(filterParser).query();
XContentType.JSON.xContent().createParser(X_CONTENT_REGISTRY, LoggingDeprecationHandler.INSTANCE, query);
QueryBuilder filter = SearchSourceBuilder.fromXContent(filterParser).query();
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().must(filter);
searchSourceBuilder
@ -403,20 +403,20 @@ public class OpenSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildPipelineSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 15.0f)
.field(Entity.FIELD_DISPLAY_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.DESCRIPTION, 1.0f)
.field(FIELD_DESCRIPTION, 1.0f)
.field("tasks.name", 2.0f)
.field("tasks.description", 1.0f)
.defaultOperator(Operator.AND)
.fuzziness(Fuzziness.AUTO);
HighlightBuilder.Field highlightPipelineName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME);
highlightPipelineName.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(EntityBuilderConstant.DESCRIPTION);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightTasks = new HighlightBuilder.Field("tasks.name");
highlightTasks.highlighterType(EntityBuilderConstant.UNIFIED);
@ -436,20 +436,20 @@ public class OpenSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildMlModelSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 15.0f)
.field(Entity.FIELD_DISPLAY_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.DESCRIPTION, 1.0f)
.field(FIELD_DESCRIPTION, 1.0f)
.field("mlFeatures.name", 2.0f)
.field("mlFeatures.description", 1.0f)
.defaultOperator(Operator.AND)
.fuzziness(Fuzziness.AUTO);
HighlightBuilder.Field highlightPipelineName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME);
highlightPipelineName.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(EntityBuilderConstant.DESCRIPTION);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightTasks = new HighlightBuilder.Field("mlFeatures.name");
highlightTasks.highlighterType(EntityBuilderConstant.UNIFIED);
@ -467,14 +467,14 @@ public class OpenSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildTopicSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 15.0f)
.field(Entity.FIELD_DISPLAY_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_NAME_NGRAM)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION, 1.0f)
.field(Entity.FIELD_DESCRIPTION, 1.0f)
.field(EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD, 2.0f)
.field("messageSchema.schemaFields.description", 1.0f)
.field("messageSchema.schemaFields.children.name", 2.0f)
@ -503,14 +503,14 @@ public class OpenSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildDashboardSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 15.0f)
.field(Entity.FIELD_DISPLAY_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_NAME_NGRAM)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION, 1.0f)
.field(Entity.FIELD_DESCRIPTION, 1.0f)
.field("charts.name", 2.0f)
.field("charts.description", 1.0f)
.defaultOperator(Operator.AND)
@ -541,13 +541,13 @@ public class OpenSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildTableSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryStringBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 15.0f)
.field(Entity.FIELD_DISPLAY_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_NAME_NGRAM)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION, 1.0f)
.field(Entity.FIELD_DESCRIPTION, 1.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.field("columns.name.keyword", 10.0f)
.field("columns.name", 2.0f)
@ -569,7 +569,7 @@ public class OpenSearchClientImpl implements SearchClient {
queryBuilder.boostMode(CombineFunction.SUM);
HighlightBuilder.Field highlightTableName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME);
highlightTableName.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(EntityBuilderConstant.DESCRIPTION);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder hb = new HighlightBuilder();
HighlightBuilder.Field highlightColumns = new HighlightBuilder.Field("columns.name");
@ -598,7 +598,7 @@ public class OpenSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildUserOrTeamSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.DISPLAY_NAME, 3.0f)
.field(FIELD_DISPLAY_NAME, 3.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 5.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 2.0f)
@ -611,16 +611,16 @@ public class OpenSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildGlossaryTermSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 10.0f)
.field(Entity.FIELD_DISPLAY_NAME, 10.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM, 1.0f)
.field(FIELD_NAME, 10.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 10.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 10.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 10.0f)
.field(Entity.FIELD_DISPLAY_NAME, 10.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field("synonyms", 5.0f)
.field("synonyms.ngram")
.field(EntityBuilderConstant.DESCRIPTION, 3.0f)
.field(FIELD_DESCRIPTION, 3.0f)
.field("glossary.name", 5.0f)
.field("glossary.displayName", 5.0f)
.field("glossary.displayName.ngram")
@ -659,9 +659,9 @@ public class OpenSearchClientImpl implements SearchClient {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(FIELD_NAME, 10.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 10.0f)
.field(Entity.FIELD_DISPLAY_NAME, 10.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM, 1.0f)
.field(EntityBuilderConstant.DESCRIPTION, 3.0f)
.field(FIELD_DESCRIPTION, 3.0f)
.defaultOperator(Operator.AND)
.fuzziness(Fuzziness.AUTO);
@ -684,10 +684,10 @@ public class OpenSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildContainerSearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME, 15.0f)
.field(Entity.FIELD_DISPLAY_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(FIELD_NAME, 15.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION, 1.0f)
.field(Entity.FIELD_DESCRIPTION, 1.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.field(EntityBuilderConstant.DISPLAY_NAME_KEYWORD, 25.0f)
.field(EntityBuilderConstant.NAME_KEYWORD, 25.0f)
@ -702,7 +702,7 @@ public class OpenSearchClientImpl implements SearchClient {
.fuzziness(Fuzziness.AUTO);
HighlightBuilder.Field highlightContainerName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME);
highlightContainerName.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(EntityBuilderConstant.DESCRIPTION);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder hb = new HighlightBuilder();
HighlightBuilder.Field highlightColumns = new HighlightBuilder.Field("dataModel.columns.name");
@ -728,16 +728,16 @@ public class OpenSearchClientImpl implements SearchClient {
private static SearchSourceBuilder buildQuerySearchBuilder(String query, int from, int size) {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(EntityBuilderConstant.DISPLAY_NAME, 10.0f)
.field(FIELD_DISPLAY_NAME, 10.0f)
.field(EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM)
.field(EntityBuilderConstant.QUERY, 10.0f)
.field(EntityBuilderConstant.QUERY_NGRAM)
.field(EntityBuilderConstant.DESCRIPTION, 1.0f)
.field(FIELD_DESCRIPTION, 1.0f)
.field(EntityBuilderConstant.FIELD_DESCRIPTION_NGRAM, 1.0f)
.defaultOperator(Operator.AND)
.fuzziness(Fuzziness.AUTO);
HighlightBuilder.Field highlightGlossaryName = new HighlightBuilder.Field(EntityBuilderConstant.DISPLAY_NAME);
HighlightBuilder.Field highlightGlossaryName = new HighlightBuilder.Field(FIELD_DISPLAY_NAME);
highlightGlossaryName.highlighterType(EntityBuilderConstant.UNIFIED);
HighlightBuilder.Field highlightDescription = new HighlightBuilder.Field(FIELD_DESCRIPTION);
highlightDescription.highlighterType(EntityBuilderConstant.UNIFIED);
@ -756,7 +756,7 @@ public class OpenSearchClientImpl implements SearchClient {
QueryStringQueryBuilder queryBuilder =
QueryBuilders.queryStringQuery(query)
.field(FIELD_NAME, 10.0f)
.field(EntityBuilderConstant.DESCRIPTION, 3.0f)
.field(FIELD_DESCRIPTION, 3.0f)
.field("testSuite.fullyQualifiedName", 10.0f)
.field("testSuite.name", 10.0f)
.field("testSuite.description", 3.0f)
@ -808,9 +808,9 @@ public class OpenSearchClientImpl implements SearchClient {
AggregationBuilders.terms("owner.displayName.keyword")
.field("owner.displayName.keyword")
.size(EntityBuilderConstant.MAX_AGGREGATE_SIZE))
.aggregation(AggregationBuilders.terms("tags.tagFQN").field("tags.tagFQN"));
;
.aggregation(
AggregationBuilders.terms(EntityBuilderConstant.ES_TAG_FQN_FIELD)
.field(EntityBuilderConstant.ES_TAG_FQN_FIELD));
return builder;
}

2
openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java
View File

@ -117,7 +117,7 @@ public class DefaultAuthorizer implements Authorizer {
}
public static SubjectContext getSubjectContext(String userName) {
return SubjectCache.getInstance().getSubjectContext(EntityInterfaceUtil.quoteName(userName));
return SubjectCache.getSubjectContext(EntityInterfaceUtil.quoteName(userName));
}
private SubjectContext changeSubjectContext(String user, SubjectContext loggedInUser) {

4
openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java
View File

@ -237,14 +237,14 @@ public class JwtFilter implements ContainerRequestFilter {
}
private void validateBotToken(String tokenFromHeader, String userName) {
if (tokenFromHeader.equals(BotTokenCache.getInstance().getToken(userName))) {
if (tokenFromHeader.equals(BotTokenCache.getToken(userName))) {
return;
}
throw AuthenticationException.getInvalidTokenException();
}
private void validatePersonalAccessToken(String tokenFromHeader, String userName) {
if (UserTokenCache.getInstance().getToken(userName).contains(tokenFromHeader)) {
if (UserTokenCache.getToken(userName).contains(tokenFromHeader)) {
return;
}
throw AuthenticationException.getInvalidTokenException();

6
openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java
View File

@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.core.Jdbi;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.Permission.Access;
import org.openmetadata.schema.type.ResourcePermission;
import org.openmetadata.service.Entity;
@ -30,15 +31,12 @@ import org.openmetadata.service.jdbi3.UserRepository;
import org.openmetadata.service.security.policyevaluator.OperationContext;
import org.openmetadata.service.security.policyevaluator.PolicyEvaluator;
import org.openmetadata.service.security.policyevaluator.ResourceContextInterface;
import org.openmetadata.service.security.policyevaluator.SubjectCache;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.RestUtil;
@Slf4j
public class NoopAuthorizer implements Authorizer {
@Override
public void init(OpenMetadataApplicationConfig openMetadataApplicationConfig, Jdbi jdbi) {
SubjectCache.initialize();
addAnonymousUser();
}
@ -68,7 +66,7 @@ public class NoopAuthorizer implements Authorizer {
private void addAnonymousUser() {
String username = "anonymous";
try {
Entity.getEntityRepository(Entity.USER).getByName(null, username, Fields.EMPTY_FIELDS);
Entity.getEntityByName(Entity.USER, username, "", Include.NON_DELETED);
} catch (EntityNotFoundException ex) {
User user =
new User()

48
openmetadata-service/src/main/java/org/openmetadata/service/security/auth/BasicAuthenticator.java
View File

@ -166,11 +166,9 @@ public class BasicAuthenticator implements AuthenticatorHandler {
String emailVerificationLink =
String.format(
"%s/users/registrationConfirmation?user=%s&token=%s",
EmailUtil.getInstance().buildBaseUrl(uriInfo.getRequestUri()),
user.getFullyQualifiedName(),
mailVerificationToken);
EmailUtil.buildBaseUrl(uriInfo.getRequestUri()), user.getFullyQualifiedName(), mailVerificationToken);
try {
EmailUtil.getInstance().sendEmailVerification(emailVerificationLink, user);
EmailUtil.sendEmailVerification(emailVerificationLink, user);
} catch (TemplateException e) {
LOG.error("Error in sending mail to the User : {}", e.getMessage(), e);
throw new CustomExceptionMessage(424, EMAIL_SENDING_ISSUE);
@ -189,11 +187,9 @@ public class BasicAuthenticator implements AuthenticatorHandler {
String passwordResetLink =
String.format(
"%s/users/password/reset?user=%s&token=%s",
EmailUtil.getInstance().buildBaseUrl(uriInfo.getRequestUri()),
user.getFullyQualifiedName(),
mailVerificationToken);
EmailUtil.buildBaseUrl(uriInfo.getRequestUri()), user.getFullyQualifiedName(), mailVerificationToken);
try {
EmailUtil.getInstance().sendPasswordResetLink(passwordResetLink, user, subject, templateFilePath);
EmailUtil.sendPasswordResetLink(passwordResetLink, user, subject, templateFilePath);
} catch (TemplateException e) {
LOG.error("Error in sending mail to the User : {}", e.getMessage(), e);
throw new CustomExceptionMessage(424, EMAIL_SENDING_ISSUE);
@ -235,7 +231,7 @@ public class BasicAuthenticator implements AuthenticatorHandler {
// Update user about Password Change
try {
EmailUtil.getInstance().sendAccountStatus(storedUser, "Update Password", "Change Successful");
EmailUtil.sendAccountStatus(storedUser, "Update Password", "Change Successful");
} catch (TemplateException ex) {
LOG.error("Error in sending Password Change Mail to User. Reason : " + ex.getMessage(), ex);
throw new CustomExceptionMessage(424, EMAIL_SENDING_ISSUE);
@ -285,7 +281,7 @@ public class BasicAuthenticator implements AuthenticatorHandler {
sendInviteMailToUser(
uriInfo,
response.getEntity(),
String.format("%s: Password Update", EmailUtil.getInstance().getEmailingEntity()),
String.format("%s: Password Update", EmailUtil.getEmailingEntity()),
ADMIN_CREATE,
request.getNewPassword());
}
@ -298,19 +294,18 @@ public class BasicAuthenticator implements AuthenticatorHandler {
switch (requestType) {
case ADMIN_CREATE:
Map<String, Object> templatePopulator = new HashMap<>();
templatePopulator.put(EmailUtil.ENTITY, EmailUtil.getInstance().getEmailingEntity());
templatePopulator.put(EmailUtil.SUPPORT_URL, EmailUtil.getInstance().getSupportUrl());
templatePopulator.put(EmailUtil.ENTITY, EmailUtil.getEmailingEntity());
templatePopulator.put(EmailUtil.SUPPORT_URL, EmailUtil.getSupportUrl());
templatePopulator.put(EmailUtil.USERNAME, user.getName());
templatePopulator.put(EmailUtil.PASSWORD, pwd);
templatePopulator.put(EmailUtil.APPLICATION_LOGIN_LINK, EmailUtil.getInstance().getOMUrl());
templatePopulator.put(EmailUtil.APPLICATION_LOGIN_LINK, EmailUtil.getOMUrl());
try {
EmailUtil.getInstance()
.sendMail(
subject,
templatePopulator,
user.getEmail(),
EmailUtil.EMAIL_TEMPLATE_BASEPATH,
EmailUtil.INVITE_RANDOM_PWD);
EmailUtil.sendMail(
subject,
templatePopulator,
user.getEmail(),
EmailUtil.EMAIL_TEMPLATE_BASEPATH,
EmailUtil.INVITE_RANDOM_PWD);
} catch (TemplateException ex) {
LOG.error("Failed in sending Mail to user [{}]. Reason : {}", user.getEmail(), ex.getMessage(), ex);
}
@ -440,13 +435,12 @@ public class BasicAuthenticator implements AuthenticatorHandler {
loginAttemptCache.recordFailedLogin(providedIdentity);
int failedLoginAttempt = loginAttemptCache.getUserFailedLoginCount(providedIdentity);
if (failedLoginAttempt == loginConfiguration.getMaxLoginFailAttempts()) {
EmailUtil.getInstance()
.sendAccountStatus(
storedUser,
"Multiple Failed Login Attempts.",
String.format(
"Someone is trying to access your account. Login is Blocked for %s minutes. Please change your password.",
loginConfiguration.getAccessBlockTime()));
EmailUtil.sendAccountStatus(
storedUser,
"Multiple Failed Login Attempts.",
String.format(
"Someone is trying to access your account. Login is Blocked for %s minutes. Please change your password.",
loginConfiguration.getAccessBlockTime()));
}
}

16
openmetadata-service/src/main/java/org/openmetadata/service/security/auth/BotTokenCache.java
View File

@ -24,11 +24,14 @@ import org.openmetadata.service.util.JsonUtils;
@Slf4j
public class BotTokenCache {
public static final String EMPTY_STRING = "";
private static BotTokenCache instance;
private static final LoadingCache<String, String> BOTS_TOKEN_CACHE =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(2, TimeUnit.MINUTES).build(new BotTokenLoader());
public String getToken(String botName) {
private BotTokenCache() {
// Private constructor for utility class
}
public static String getToken(String botName) {
try {
if (BOTS_TOKEN_CACHE.get(botName).equals(EMPTY_STRING)) {
BOTS_TOKEN_CACHE.invalidate(botName);
@ -39,7 +42,7 @@ public class BotTokenCache {
}
}
public void invalidateToken(String botName) {
public static void invalidateToken(String botName) {
try {
BOTS_TOKEN_CACHE.invalidate(botName);
} catch (Exception ex) {
@ -47,13 +50,6 @@ public class BotTokenCache {
}
}
public static BotTokenCache getInstance() {
if (instance == null) {
instance = new BotTokenCache();
}
return instance;
}
static class BotTokenLoader extends CacheLoader<String, String> {
@Override
public String load(@CheckForNull String botName) throws IOException {

13
openmetadata-service/src/main/java/org/openmetadata/service/security/auth/LdapAuthenticator.java
View File

@ -143,13 +143,12 @@ public class LdapAuthenticator implements AuthenticatorHandler {
loginAttemptCache.recordFailedLogin(providedIdentity);
int failedLoginAttempt = loginAttemptCache.getUserFailedLoginCount(providedIdentity);
if (failedLoginAttempt == loginConfiguration.getMaxLoginFailAttempts()) {
EmailUtil.getInstance()
.sendAccountStatus(
storedUser,
"Multiple Failed Login Attempts.",
String.format(
"Someone is tried accessing your account. Login is Blocked for %s seconds.",
loginConfiguration.getAccessBlockTime()));
EmailUtil.sendAccountStatus(
storedUser,
"Multiple Failed Login Attempts.",
String.format(
"Someone is tried accessing your account. Login is Blocked for %s seconds.",
loginConfiguration.getAccessBlockTime()));
}
}

1
openmetadata-service/src/main/java/org/openmetadata/service/security/auth/LoginAttemptCache.java
View File

@ -14,7 +14,6 @@ public class LoginAttemptCache {
private final LoadingCache<String, Integer> attemptsCache;
public LoginAttemptCache(OpenMetadataApplicationConfig config) {
super();
LoginConfiguration loginConfiguration = config.getApplicationConfiguration().getLoginConfig();
long accessBlockTime = 600;
if (loginConfiguration != null) {

15
openmetadata-service/src/main/java/org/openmetadata/service/security/auth/UserTokenCache.java
View File

@ -27,8 +27,7 @@ import org.openmetadata.service.util.EntityUtil.Fields;
@Slf4j
public class UserTokenCache {
private static final UserTokenCache instance = new UserTokenCache();
private static final LoadingCache<String, HashSet<String>> cache =
private static final LoadingCache<String, HashSet<String>> CACHE =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(2, TimeUnit.MINUTES).build(new UserTokenLoader());
private static volatile boolean initialized = false;
private static TokenRepository tokenRepository;
@ -47,27 +46,23 @@ public class UserTokenCache {
}
}
public Set<String> getToken(String userName) {
public static Set<String> getToken(String userName) {
try {
return cache.get(userName);
return CACHE.get(userName);
} catch (ExecutionException | UncheckedExecutionException ex) {
LOG.error("Token not found", ex);
return null;
}
}
public void invalidateToken(String userName) {
public static void invalidateToken(String userName) {
try {
cache.invalidate(userName);
CACHE.invalidate(userName);
} catch (Exception ex) {
LOG.error("Failed to invalidate User token cache for User {}", userName, ex);
}
}
public static UserTokenCache getInstance() {
return instance;
}
static class UserTokenLoader extends CacheLoader<String, HashSet<String>> {
@Override
public HashSet<String> load(@CheckForNull String userName) throws IOException {

43
openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/PolicyCache.java
View File

@ -27,53 +27,33 @@ import javax.annotation.CheckForNull;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.entity.policies.Policy;
import org.openmetadata.schema.entity.policies.accessControl.Rule;
import org.openmetadata.schema.type.Include;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.PolicyRepository;
import org.openmetadata.service.util.EntityUtil.Fields;
/** Subject context used for Access Control Policies */
@Slf4j
public class PolicyCache {
private static final PolicyCache INSTANCE = new PolicyCache();
private static volatile boolean initialized = false;
protected static final LoadingCache<UUID, List<CompiledRule>> CACHE =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new PolicyLoader());
protected static LoadingCache<UUID, List<CompiledRule>> policyCache;
private static PolicyRepository policyRepository;
private static Fields fields;
public static PolicyCache getInstance() {
return INSTANCE;
}
/** To be called during application startup by Default Authorizer */
public static void initialize() {
if (!initialized) {
policyCache =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new PolicyLoader());
policyRepository = (PolicyRepository) Entity.getEntityRepository(Entity.POLICY);
fields = policyRepository.getFields("rules");
initialized = true;
}
}
public List<CompiledRule> getPolicyRules(UUID policyId) {
public static List<CompiledRule> getPolicyRules(UUID policyId) {
try {
return policyCache.get(policyId);
return CACHE.get(policyId);
} catch (ExecutionException | UncheckedExecutionException ex) {
throw new EntityNotFoundException(ex.getMessage());
}
}
public void invalidatePolicy(UUID policyId) {
public static void invalidatePolicy(UUID policyId) {
try {
policyCache.invalidate(policyId);
CACHE.invalidate(policyId);
} catch (Exception ex) {
LOG.error("Failed to invalidate cache for policy {}", policyId, ex);
}
}
protected List<CompiledRule> getRules(Policy policy) {
protected static List<CompiledRule> getRules(Policy policy) {
List<CompiledRule> rules = new ArrayList<>();
for (Rule r : policy.getRules()) {
rules.add(new CompiledRule(r));
@ -82,16 +62,15 @@ public class PolicyCache {
}
public static void cleanUp() {
policyCache.cleanUp();
initialized = false;
CACHE.cleanUp();
}
static class PolicyLoader extends CacheLoader<UUID, List<CompiledRule>> {
@Override
public List<CompiledRule> load(@CheckForNull UUID policyId) throws IOException {
Policy policy = policyRepository.get(null, policyId, fields);
Policy policy = Entity.getEntity(Entity.POLICY, policyId, "rules", Include.NON_DELETED);
LOG.info("Loaded policy {}:{}", policy.getName(), policy.getId());
return PolicyCache.getInstance().getRules(policy);
return PolicyCache.getRules(policy);
}
}
}

2
openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/PostResourceContext.java
View File

@ -22,7 +22,7 @@ public class PostResourceContext implements ResourceContextInterface {
@Override
public EntityReference getOwner() {
return SubjectCache.getInstance().getUser(postedBy).getEntityReference();
return SubjectCache.getUser(postedBy).getEntityReference();
}
@Override

52
openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/RoleCache.java
View File

@ -26,60 +26,41 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.CheckForNull;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.entity.teams.Role;
import org.openmetadata.schema.type.Include;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.RoleRepository;
import org.openmetadata.service.util.EntityUtil.Fields;
/** Subject context used for Access Control Policies */
@Slf4j
public class RoleCache {
private static final RoleCache INSTANCE = new RoleCache();
private static volatile boolean initialized = false;
protected static LoadingCache<String, Role> roleCache;
protected static LoadingCache<UUID, Role> roleCacheWithId;
private static RoleRepository roleRepository;
private static Fields fields;
protected static final LoadingCache<String, Role> CACHE =
CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(3, TimeUnit.MINUTES).build(new RoleLoader());
protected static final LoadingCache<UUID, Role> CACHE_WITH_ID =
CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(3, TimeUnit.MINUTES).build(new RoleLoaderWithId());
public static RoleCache getInstance() {
return INSTANCE;
private RoleCache() {
// Private constructor for singleton
}
/** To be called only once during the application start from DefaultAuthorizer */
public static void initialize() {
if (!initialized) {
roleCache =
CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(3, TimeUnit.MINUTES).build(new RoleLoader());
roleCacheWithId =
CacheBuilder.newBuilder()
.maximumSize(100)
.expireAfterWrite(3, TimeUnit.MINUTES)
.build(new RoleLoaderWithId());
roleRepository = (RoleRepository) Entity.getEntityRepository(Entity.ROLE);
fields = roleRepository.getFields("policies");
initialized = true;
}
}
public Role getRole(String roleName) {
public static Role getRole(String roleName) {
try {
return roleCache.get(roleName);
return CACHE.get(roleName);
} catch (ExecutionException | UncheckedExecutionException ex) {
throw EntityNotFoundException.byMessage(entityNotFound(Entity.ROLE, roleName));
}
}
public Role getRoleById(UUID roleId) {
public static Role getRoleById(UUID roleId) {
try {
return roleCacheWithId.get(roleId);
return CACHE_WITH_ID.get(roleId);
} catch (ExecutionException | UncheckedExecutionException ex) {
throw EntityNotFoundException.byMessage(entityNotFound(Entity.ROLE, roleId));
}
}
public void invalidateRole(UUID roleId) {
public static void invalidateRole(UUID roleId) {
try {
roleCacheWithId.invalidate(roleId);
CACHE_WITH_ID.invalidate(roleId);
} catch (Exception ex) {
LOG.error("Failed to invalidate cache for role {}", roleId, ex);
}
@ -88,7 +69,7 @@ public class RoleCache {
static class RoleLoader extends CacheLoader<String, Role> {
@Override
public Role load(@CheckForNull String roleName) throws IOException {
Role role = roleRepository.getByName(null, roleName, fields);
Role role = Entity.getEntityByName(Entity.ROLE, roleName, "policies", Include.NON_DELETED);
LOG.info("Loaded role {}:{}", role.getName(), role.getId());
return role;
}
@ -97,14 +78,13 @@ public class RoleCache {
static class RoleLoaderWithId extends CacheLoader<UUID, Role> {
@Override
public Role load(@CheckForNull UUID roleId) throws IOException {
Role role = roleRepository.get(null, roleId, fields);
Role role = Entity.getEntity(Entity.ROLE, roleId, "policies", Include.NON_DELETED);
LOG.info("Loaded role {}:{}", role.getName(), role.getId());
return role;
}
}
public static void cleanUp() {
roleCacheWithId.cleanUp();
initialized = false;
CACHE_WITH_ID.cleanUp();
}
}

8
openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/RuleEvaluator.java
View File

@ -75,7 +75,7 @@ public class RuleEvaluator {
public boolean matchAllTags(String... tagFQNs) throws IOException {
if (expressionValidation) {
for (String tagFqn : tagFQNs) {
TagLabelCache.getInstance().getTag(tagFqn);
TagLabelCache.getTag(tagFqn);
}
return false;
}
@ -102,7 +102,7 @@ public class RuleEvaluator {
public boolean matchAnyTag(String... tagFQNs) throws IOException {
if (expressionValidation) {
for (String tagFqn : tagFQNs) {
TagLabelCache.getInstance().getTag(tagFqn);
TagLabelCache.getTag(tagFqn);
}
return false;
}
@ -151,7 +151,7 @@ public class RuleEvaluator {
public boolean inAnyTeam(String... teams) {
if (expressionValidation) {
for (String team : teams) {
SubjectCache.getInstance().getTeamByName(team);
SubjectCache.getTeamByName(team);
}
return false;
}
@ -179,7 +179,7 @@ public class RuleEvaluator {
public boolean hasAnyRole(String... roles) {
if (expressionValidation) {
for (String role : roles) {
RoleCache.getInstance().getRole(role);
RoleCache.getRole(role);
}
return false;
}

117
openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/SubjectCache.java
View File

@ -14,6 +14,7 @@
package org.openmetadata.service.security.policyevaluator;
import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
import static org.openmetadata.schema.type.Include.NON_DELETED;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
@ -38,111 +39,79 @@ import org.openmetadata.schema.utils.EntityInterfaceUtil;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.CatalogExceptionMessage;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.TeamRepository;
import org.openmetadata.service.jdbi3.UserRepository;
import org.openmetadata.service.util.EntityUtil.Fields;
/** Subject context used for Access Control Policies */
@Slf4j
public class SubjectCache {
private static SubjectCache instance;
private static volatile boolean initialized = false;
protected static LoadingCache<String, SubjectContext> userCache;
protected static LoadingCache<UUID, SubjectContext> userCacheWihId;
protected static LoadingCache<String, Team> teamCache;
protected static LoadingCache<UUID, Team> teamCacheWithId;
protected static UserRepository userRepository;
protected static Fields userFields;
protected static TeamRepository teamRepository;
protected static Fields teamFields;
protected static final LoadingCache<String, SubjectContext> USER_CACHE =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new UserLoader());
protected static final LoadingCache<UUID, SubjectContext> USER_CACHE_WITH_ID =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new UserLoaderWithId());
protected static final LoadingCache<String, Team> TEAM_CACHE =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new TeamLoader());
protected static final LoadingCache<UUID, Team> TEAM_CACHE_WITH_ID =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new TeamLoaderWithId());
private static final String USER_FIELDS = "roles,teams,isAdmin,profile";
private static final String TEAM_FIELDS = "defaultRoles, policies, parents, profile";
// Expected to be called only once from the DefaultAuthorizer
public static void initialize() {
if (!initialized) {
userCache =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new UserLoader());
userCacheWihId =
CacheBuilder.newBuilder()
.maximumSize(1000)
.expireAfterWrite(3, TimeUnit.MINUTES)
.build(new UserLoaderWithId());
teamCache =
CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(3, TimeUnit.MINUTES).build(new TeamLoader());
teamCacheWithId =
CacheBuilder.newBuilder()
.maximumSize(1000)
.expireAfterWrite(3, TimeUnit.MINUTES)
.build(new TeamLoaderWithId());
userRepository = (UserRepository) Entity.getEntityRepository(Entity.USER);
userFields = userRepository.getFields("roles, teams, isAdmin, profile");
teamRepository = (TeamRepository) Entity.getEntityRepository(Entity.TEAM);
teamFields = teamRepository.getFields("defaultRoles, policies, parents, profile");
instance = new SubjectCache();
initialized = true;
LOG.info("Subject cache is initialized");
} else {
LOG.info("Subject cache is already initialized");
}
private SubjectCache() {
// Private constructor for singleton
}
public static SubjectCache getInstance() {
return instance;
}
public SubjectContext getSubjectContext(String userName) throws EntityNotFoundException {
public static SubjectContext getSubjectContext(String userName) throws EntityNotFoundException {
try {
return userCache.get(userName);
return USER_CACHE.get(userName);
} catch (ExecutionException | UncheckedExecutionException ex) {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Entity.USER, userName));
}
}
public SubjectContext getSubjectContext(UUID userId) throws EntityNotFoundException {
public static SubjectContext getSubjectContext(UUID userId) throws EntityNotFoundException {
try {
return userCacheWihId.get(userId);
return USER_CACHE_WITH_ID.get(userId);
} catch (ExecutionException | UncheckedExecutionException ex) {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Entity.USER, userId));
}
}
public User getUser(String userName) throws EntityNotFoundException {
public static User getUser(String userName) throws EntityNotFoundException {
try {
return userCache.get(userName).getUser();
return USER_CACHE.get(userName).getUser();
} catch (ExecutionException | UncheckedExecutionException ex) {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Entity.USER, userName));
}
}
public User getUserById(String userId) throws EntityNotFoundException {
public static User getUserById(String userId) throws EntityNotFoundException {
return getUserById(UUID.fromString(userId));
}
public User getUserById(UUID userId) throws EntityNotFoundException {
public static User getUserById(UUID userId) throws EntityNotFoundException {
try {
return userCacheWihId.get(userId).getUser();
return USER_CACHE_WITH_ID.get(userId).getUser();
} catch (ExecutionException | UncheckedExecutionException ex) {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Entity.USER, userId));
}
}
public Team getTeam(UUID teamId) throws EntityNotFoundException {
public static Team getTeam(UUID teamId) throws EntityNotFoundException {
try {
return teamCacheWithId.get(teamId);
return TEAM_CACHE_WITH_ID.get(teamId);
} catch (ExecutionException | UncheckedExecutionException ex) {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Entity.TEAM, teamId));
}
}
public Team getTeamByName(String teamName) throws EntityNotFoundException {
public static Team getTeamByName(String teamName) throws EntityNotFoundException {
try {
return teamCache.get(teamName);
return TEAM_CACHE.get(teamName);
} catch (ExecutionException | UncheckedExecutionException ex) {
throw EntityNotFoundException.byMessage(CatalogExceptionMessage.entityNotFound(Entity.TEAM, teamName));
}
}
/** Return true if given list of teams is part of the hierarchy of parentTeam */
public boolean isInTeam(String parentTeam, EntityReference team) {
public static boolean isInTeam(String parentTeam, EntityReference team) {
Deque<EntityReference> stack = new ArrayDeque<>();
stack.push(team); // Start with team and see if the parent matches
while (!stack.isEmpty()) {
@ -156,7 +125,7 @@ public class SubjectCache {
}
/** Return true if the given user has any roles the list of roles */
public boolean hasRole(User user, String role) {
public static boolean hasRole(User user, String role) {
Deque<EntityReference> stack = new ArrayDeque<>();
// If user has one of the roles directly assigned then return true
if (hasRole(user.getRoles(), role)) {
@ -179,28 +148,27 @@ public class SubjectCache {
public static void cleanUp() {
LOG.info("Subject cache is cleaned up");
userCache.invalidateAll();
teamCacheWithId.invalidateAll();
initialized = false;
USER_CACHE.invalidateAll();
TEAM_CACHE_WITH_ID.invalidateAll();
}
public void invalidateUser(String userName) {
public static void invalidateUser(String userName) {
try {
userCache.invalidate(userName);
USER_CACHE.invalidate(userName);
} catch (Exception ex) {
LOG.error("Failed to invalidate cache for user {}", userName, ex);
}
}
public void invalidateTeam(UUID teamId) {
public static void invalidateTeam(UUID teamId) {
try {
teamCacheWithId.invalidate(teamId);
TEAM_CACHE_WITH_ID.invalidate(teamId);
} catch (Exception ex) {
LOG.error("Failed to invalidate cache for team {}", teamId, ex);
}
}
public List<EntityReference> getRolesForTeams(List<EntityReference> teams) {
public static List<EntityReference> getRolesForTeams(List<EntityReference> teams) {
List<EntityReference> roles = new ArrayList<>();
for (EntityReference teamRef : listOrEmpty(teams)) {
Team team = getTeam(teamRef.getId());
@ -215,10 +183,9 @@ public class SubjectCache {
static class UserLoader extends CacheLoader<String, SubjectContext> {
@Override
public SubjectContext load(@CheckForNull String userName) throws IOException {
// XXX
try {
System.out.println("Loading user by name " + userName);
User user = userRepository.getByName(null, EntityInterfaceUtil.quoteName(userName), userFields);
User user =
Entity.getEntityByName(Entity.USER, EntityInterfaceUtil.quoteName(userName), USER_FIELDS, NON_DELETED);
LOG.info("Loaded user {}:{}", user.getName(), user.getId());
return new SubjectContext(user);
} catch (Exception ex) {
@ -231,7 +198,7 @@ public class SubjectCache {
static class UserLoaderWithId extends CacheLoader<UUID, SubjectContext> {
@Override
public SubjectContext load(@CheckForNull UUID uid) throws IOException {
User user = userRepository.get(null, uid, userFields);
User user = Entity.getEntity(Entity.USER, uid, USER_FIELDS, NON_DELETED);
LOG.info("Loaded user {}:{}", user.getName(), user.getId());
return new SubjectContext(user);
}
@ -239,9 +206,9 @@ public class SubjectCache {
static class TeamLoader extends CacheLoader<String, Team> {
@Override
public Team load(@CheckForNull String userName) throws IOException {
Team team = teamRepository.getByName(null, userName, teamFields);
LOG.info("Loaded user {}:{}", team.getName(), team.getId());
public Team load(@CheckForNull String teamName) throws IOException {
Team team = Entity.getEntityByName(Entity.TEAM, teamName, TEAM_FIELDS, NON_DELETED);
LOG.info("Loaded team {}:{}", team.getName(), team.getId());
return team;
}
}
@ -249,7 +216,7 @@ public class SubjectCache {
static class TeamLoaderWithId extends CacheLoader<UUID, Team> {
@Override
public Team load(@NonNull UUID teamId) throws IOException {
Team team = teamRepository.get(null, teamId, teamFields);
Team team = Entity.getEntity(Entity.TEAM, teamId, TEAM_FIELDS, NON_DELETED);
LOG.info("Loaded team {}:{}", team.getName(), team.getId());
return team;
}

19
openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/SubjectContext.java
View File

@ -74,10 +74,10 @@ public class SubjectContext {
/** Returns true if the given resource owner is under the team hierarchy of parentTeam */
public boolean isTeamAsset(String parentTeam, EntityReference owner) {
if (owner.getType().equals(Entity.USER)) {
SubjectContext subjectContext = SubjectCache.getInstance().getSubjectContext(owner.getName());
SubjectContext subjectContext = SubjectCache.getSubjectContext(owner.getName());
return subjectContext.isUserUnderTeam(parentTeam);
} else if (owner.getType().equals(Entity.TEAM)) {
Team team = SubjectCache.getInstance().getTeam(owner.getId());
Team team = SubjectCache.getTeam(owner.getId());
return isInTeam(parentTeam, team.getEntityReference());
}
return false;
@ -85,7 +85,7 @@ public class SubjectContext {
/** Return true if the team is part of the hierarchy of parentTeam */
private boolean isInTeam(String parentTeam, EntityReference team) {
return SubjectCache.getInstance().isInTeam(parentTeam, team);
return SubjectCache.isInTeam(parentTeam, team);
}
// Iterate over all the policies of the team hierarchy the user belongs to
@ -99,7 +99,7 @@ public class SubjectContext {
/** Returns true if the user has any of the roles (either direct or inherited roles) */
public boolean hasAnyRole(String roles) {
return SubjectCache.getInstance().hasRole(getUser(), roles);
return SubjectCache.hasRole(getUser(), roles);
}
@Getter
@ -163,7 +163,7 @@ public class SubjectContext {
}
EntityReference policy = policies.get(policyIndex++);
return new PolicyContext(
entityType, entityName, roleName, policy.getName(), PolicyCache.getInstance().getPolicyRules(policy.getId()));
entityType, entityName, roleName, policy.getName(), PolicyCache.getPolicyRules(policy.getId()));
}
}
@ -184,10 +184,7 @@ public class SubjectContext {
for (EntityReference role : listOrEmpty(roles)) {
policyIterators.add(
new PolicyIterator(
entityType,
entityName,
role.getName(),
RoleCache.getInstance().getRoleById(role.getId()).getPolicies()));
entityType, entityName, role.getName(), RoleCache.getRoleById(role.getId()).getPolicies()));
}
}
@ -240,7 +237,7 @@ public class SubjectContext {
// Finally, iterate over policies of teams that own the resource
if (resourceOwner != null && resourceOwner.getType().equals(Entity.TEAM)) {
Team team = SubjectCache.getInstance().getTeam(resourceOwner.getId());
Team team = SubjectCache.getTeam(resourceOwner.getId());
iterators.add(new TeamPolicyIterator(team.getId(), teamsVisited, true));
}
}
@ -276,7 +273,7 @@ public class SubjectContext {
/** Policy iterator for a team */
TeamPolicyIterator(UUID teamId, List<UUID> teamsVisited, boolean skipRoles) {
Team team = SubjectCache.getInstance().getTeam(teamId);
Team team = SubjectCache.getTeam(teamId);
// If a team is already visited (because user can belong to multiple teams
// and a team can belong to multiple teams) then don't visit the roles/policies of that team

2
openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/ThreadResourceContext.java
View File

@ -22,7 +22,7 @@ public class ThreadResourceContext implements ResourceContextInterface {
@Override
public EntityReference getOwner() {
return SubjectCache.getInstance().getUser(createdBy).getEntityReference();
return SubjectCache.getUser(createdBy).getEntityReference();
}
@Override

2
openmetadata-service/src/main/java/org/openmetadata/service/socket/WebSocketManager.java
View File

@ -102,7 +102,7 @@ public class WebSocketManager {
public void sendToOne(String username, String event, String message) {
try {
UUID receiver = SubjectCache.getInstance().getSubjectContext(username).getUser().getId();
UUID receiver = SubjectCache.getSubjectContext(username).getUser().getId();
if (activityFeedEndpoints.containsKey(receiver)) {
activityFeedEndpoints.get(receiver).forEach((key, value) -> value.send(event, message));
}

142
openmetadata-service/src/main/java/org/openmetadata/service/util/EmailUtil.java
View File

@ -35,7 +35,6 @@ import org.openmetadata.schema.email.SmtpSettings;
import org.openmetadata.schema.entity.feed.Thread;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.settings.SettingsType;
import org.openmetadata.schema.tests.type.TestCaseResult;
import org.openmetadata.service.events.scheduled.template.DataInsightDescriptionAndOwnerTemplate;
import org.openmetadata.service.events.scheduled.template.DataInsightTotalAssetTemplate;
import org.openmetadata.service.events.subscription.email.EmailMessage;
@ -74,28 +73,23 @@ public class EmailUtil {
private static final String CHANGE_EVENT_UPDATE = "Change Event Update from %s";
private static final String TASK_SUBJECT = "%s : Task Assignment Notification";
private static final String TEST_SUBJECT = "%s : Test Result Notification";
public static final String INVITE_RANDOM_PWD = "invite-randompwd.ftl";
public static final String CHANGE_EVENT_TEMPLATE = "changeEvent.ftl";
public static final String INVITE_CREATE_PWD = "invite-createPassword.ftl";
public static final String TASK_NOTIFICATION_TEMPLATE = "taskAssignment.ftl";
public static final String TEST_NOTIFICATION_TEMPLATE = "testResultStatus.ftl";
private static final String REPORT_SUBJECT = "%s: Data Insights Weekly - %s";
public static final String DATA_INSIGHT_REPORT_TEMPLATE = "dataInsightReport.ftl";
private static EmailUtil instance;
private static SmtpSettings STORED_SMTP_SETTINGS;
private static SmtpSettings storedSmtpSettings;
private static Mailer mailer;
private static Configuration templateConfiguration;
private static final Configuration templateConfiguration = new Configuration(VERSION_2_3_28);
private static final String EMAIL_IGNORE_MSG = "Email was not sent to %s as SMTP setting is not enabled";
private static final String EMAIL_IGNORE_MSG = "Email was not sent to {} as SMTP setting is not enabled";
private EmailUtil() {
try {
STORED_SMTP_SETTINGS = getSmtpSettings();
mailer = createMailer(STORED_SMTP_SETTINGS);
templateConfiguration = new Configuration(VERSION_2_3_28);
getSmtpSettings();
LOG.info("Email Util cache is initialized");
} catch (Exception ex) {
LOG.warn("[MAILER] Smtp Configurations are missing : Reason {} ", ex.getMessage(), ex);
@ -127,14 +121,7 @@ public class EmailUtil {
return null;
}
public static EmailUtil getInstance() {
if (instance == null) {
instance = new EmailUtil();
}
return instance;
}
public void sendAccountStatus(User user, String action, String status) throws IOException, TemplateException {
public static void sendAccountStatus(User user, String action, String status) throws IOException, TemplateException {
if (Boolean.TRUE.equals(getSmtpSettings().getEnableSmtpServer())) {
Map<String, Object> templatePopulator = new HashMap<>();
templatePopulator.put(ENTITY, getEmailingEntity());
@ -149,11 +136,12 @@ public class EmailUtil {
EMAIL_TEMPLATE_BASEPATH,
ACCOUNT_STATUS_TEMPLATE_FILE);
} else {
LOG.warn(String.format(EMAIL_IGNORE_MSG, user.getEmail()));
LOG.warn(EMAIL_IGNORE_MSG, user.getEmail());
}
}
public void sendEmailVerification(String emailVerificationLink, User user) throws IOException, TemplateException {
public static void sendEmailVerification(String emailVerificationLink, User user)
throws IOException, TemplateException {
if (Boolean.TRUE.equals(getSmtpSettings().getEnableSmtpServer())) {
Map<String, Object> templatePopulator = new HashMap<>();
templatePopulator.put(ENTITY, getEmailingEntity());
@ -168,11 +156,11 @@ public class EmailUtil {
EMAIL_TEMPLATE_BASEPATH,
EMAIL_VERIFICATION_TEMPLATE_PATH);
} else {
LOG.warn(String.format(EMAIL_IGNORE_MSG, user.getEmail()));
LOG.warn(EMAIL_IGNORE_MSG, user.getEmail());
}
}
public void sendPasswordResetLink(String passwordResetLink, User user, String subject, String templateFilePath)
public static void sendPasswordResetLink(String passwordResetLink, User user, String subject, String templateFilePath)
throws IOException, TemplateException {
if (Boolean.TRUE.equals(getSmtpSettings().getEnableSmtpServer())) {
Map<String, Object> templatePopulator = new HashMap<>();
@ -184,11 +172,11 @@ public class EmailUtil {
sendMail(subject, templatePopulator, user.getEmail(), EMAIL_TEMPLATE_BASEPATH, templateFilePath);
} else {
LOG.warn(String.format(EMAIL_IGNORE_MSG, user.getEmail()));
LOG.warn(EMAIL_IGNORE_MSG, user.getEmail());
}
}
public void sendTaskAssignmentNotificationToUser(
public static void sendTaskAssignmentNotificationToUser(
String assigneeName, String email, String taskLink, Thread thread, String subject, String templateFilePath)
throws IOException, TemplateException {
if (Boolean.TRUE.equals(getSmtpSettings().getEnableSmtpServer())) {
@ -204,32 +192,11 @@ public class EmailUtil {
sendMail(subject, templatePopulator, email, EMAIL_TEMPLATE_BASEPATH, templateFilePath);
} else {
LOG.warn(String.format(EMAIL_IGNORE_MSG, email));
LOG.warn(EMAIL_IGNORE_MSG, email);
}
}
public void sendTestResultEmailNotificationToUser(
String email,
String testResultLink,
String testCaseName,
TestCaseResult result,
String subject,
String templateFilePath)
throws IOException, TemplateException {
if (Boolean.TRUE.equals(getSmtpSettings().getEnableSmtpServer())) {
Map<String, Object> templatePopulator = new HashMap<>();
templatePopulator.put("receiverName", email.split("@")[0]);
templatePopulator.put("testResultName", testCaseName);
templatePopulator.put("testResultDescription", result.getResult());
templatePopulator.put("testResultStatus", result.getTestCaseStatus().toString());
templatePopulator.put("testResultTimestamp", result.getTimestamp().toString());
templatePopulator.put("testResultLink", testResultLink);
sendMail(subject, templatePopulator, email, EMAIL_TEMPLATE_BASEPATH, templateFilePath);
}
}
public void sendMail(
public static void sendMail(
String subject, Map<String, Object> model, String to, String baseTemplatePackage, String templatePath)
throws IOException, TemplateException {
if (Boolean.TRUE.equals(getSmtpSettings().getEnableSmtpServer())) {
@ -238,7 +205,7 @@ public class EmailUtil {
emailBuilder.to(to);
emailBuilder.from(getSmtpSettings().getSenderMail());
templateConfiguration.setClassForTemplateLoading(getClass(), baseTemplatePackage);
templateConfiguration.setClassForTemplateLoading(EmailUtil.class, baseTemplatePackage);
Template template = templateConfiguration.getTemplate(templatePath);
// write the freemarker output to a StringWriter
@ -252,7 +219,7 @@ public class EmailUtil {
}
}
public void sendMailToMultiple(
public static void sendMailToMultiple(
String subject, Map<String, Object> model, Set<String> to, String baseTemplatePackage, String templatePath)
throws IOException, TemplateException {
if (Boolean.TRUE.equals(getSmtpSettings().getEnableSmtpServer())) {
@ -261,7 +228,7 @@ public class EmailUtil {
emailBuilder.toMultiple(to);
emailBuilder.from(getSmtpSettings().getSenderMail());
templateConfiguration.setClassForTemplateLoading(getClass(), baseTemplatePackage);
templateConfiguration.setClassForTemplateLoading(EmailUtil.class, baseTemplatePackage);
Template template = templateConfiguration.getTemplate(templatePath);
// write the freemarker output to a StringWriter
@ -273,13 +240,13 @@ public class EmailUtil {
}
}
public void sendMail(Email email) {
public static void sendMail(Email email) {
if (mailer != null && getSmtpSettings().getEnableSmtpServer()) {
mailer.sendMail(email, true);
}
}
public String buildBaseUrl(URI uri) {
public static String buildBaseUrl(URI uri) {
try {
if (CommonUtil.nullOrEmpty(getSmtpSettings().getOpenMetadataUrl())) {
return String.format("%s://%s", uri.getScheme(), uri.getHost());
@ -295,24 +262,23 @@ public class EmailUtil {
public static void sendInviteMailToAdmin(User user, String pwd) {
if (Boolean.TRUE.equals(getSmtpSettings().getEnableSmtpServer())) {
Map<String, Object> templatePopulator = new HashMap<>();
templatePopulator.put(EmailUtil.ENTITY, EmailUtil.getInstance().getEmailingEntity());
templatePopulator.put(EmailUtil.SUPPORT_URL, EmailUtil.getInstance().getSupportUrl());
templatePopulator.put(EmailUtil.ENTITY, EmailUtil.getEmailingEntity());
templatePopulator.put(EmailUtil.SUPPORT_URL, EmailUtil.getSupportUrl());
templatePopulator.put(EmailUtil.USERNAME, user.getName());
templatePopulator.put(EmailUtil.PASSWORD, pwd);
templatePopulator.put(EmailUtil.APPLICATION_LOGIN_LINK, EmailUtil.getInstance().getOMUrl());
templatePopulator.put(EmailUtil.APPLICATION_LOGIN_LINK, EmailUtil.getOMUrl());
try {
EmailUtil.getInstance()
.sendMail(
EmailUtil.getInstance().getEmailInviteSubject(),
templatePopulator,
user.getEmail(),
EmailUtil.EMAIL_TEMPLATE_BASEPATH,
EmailUtil.INVITE_RANDOM_PWD);
EmailUtil.sendMail(
EmailUtil.getEmailInviteSubject(),
templatePopulator,
user.getEmail(),
EmailUtil.EMAIL_TEMPLATE_BASEPATH,
EmailUtil.INVITE_RANDOM_PWD);
} catch (Exception ex) {
LOG.error("Failed in sending Mail to user [{}]. Reason : {}", user.getEmail(), ex.getMessage());
}
} else {
LOG.warn(String.format(EMAIL_IGNORE_MSG, user.getEmail()));
LOG.warn(EMAIL_IGNORE_MSG, user.getEmail());
}
}
@ -329,22 +295,21 @@ public class EmailUtil {
}
templatePopulator.put("changeMessage", buff.toString());
try {
EmailUtil.getInstance()
.sendMail(
EmailUtil.getInstance().getChangeEventTemplate(),
templatePopulator,
receiverMail,
EmailUtil.EMAIL_TEMPLATE_BASEPATH,
EmailUtil.CHANGE_EVENT_TEMPLATE);
EmailUtil.sendMail(
EmailUtil.getChangeEventTemplate(),
templatePopulator,
receiverMail,
EmailUtil.EMAIL_TEMPLATE_BASEPATH,
EmailUtil.CHANGE_EVENT_TEMPLATE);
} catch (Exception ex) {
LOG.error("Failed in sending Mail to user [{}]. Reason : {}", receiverMail, ex.getMessage());
}
} else {
LOG.warn(String.format(EMAIL_IGNORE_MSG, receiverMail));
LOG.warn(EMAIL_IGNORE_MSG, receiverMail);
}
}
public void sendDataInsightEmailNotificationToUser(
public static void sendDataInsightEmailNotificationToUser(
Set<String> emails,
DataInsightTotalAssetTemplate totalAssetObj,
DataInsightDescriptionAndOwnerTemplate descriptionObj,
@ -365,60 +330,55 @@ public class EmailUtil {
}
}
public void testConnection() {
public static void testConnection() {
mailer.testConnection();
}
private String getEmailVerificationSubject() {
private static String getEmailVerificationSubject() {
return String.format(EMAIL_VERIFICATION_SUBJECT, getSmtpSettings().getEmailingEntity());
}
public String getPasswordResetSubject() {
public static String getPasswordResetSubject() {
return String.format(PASSWORD_RESET_SUBJECT, getSmtpSettings().getEmailingEntity());
}
private String getAccountStatusChangeSubject() {
private static String getAccountStatusChangeSubject() {
return String.format(ACCOUNT_STATUS_SUBJECT, getSmtpSettings().getEmailingEntity());
}
public String getEmailInviteSubject() {
public static String getEmailInviteSubject() {
return String.format(INVITE_SUBJECT, getSmtpSettings().getEmailingEntity());
}
public String getChangeEventTemplate() {
public static String getChangeEventTemplate() {
return String.format(CHANGE_EVENT_UPDATE, getSmtpSettings().getEmailingEntity());
}
public String getTaskAssignmentSubject() {
public static String getTaskAssignmentSubject() {
return String.format(TASK_SUBJECT, getSmtpSettings().getEmailingEntity());
}
public String getTestResultSubject() {
return String.format(TEST_SUBJECT, getSmtpSettings().getEmailingEntity());
}
public String getDataInsightReportSubject() {
public static String getDataInsightReportSubject() {
return String.format(
REPORT_SUBJECT, getSmtpSettings().getEmailingEntity(), new SimpleDateFormat("dd-MM-yy").format(new Date()));
}
public String getEmailingEntity() {
public static String getEmailingEntity() {
return getSmtpSettings().getEmailingEntity();
}
public String getSupportUrl() {
public static String getSupportUrl() {
return getSmtpSettings().getSupportUrl();
}
public String getOMUrl() {
public static String getOMUrl() {
return getSmtpSettings().getOpenMetadataUrl();
}
private static SmtpSettings getSmtpSettings() {
SmtpSettings emailConfig =
SettingsCache.getInstance().getSetting(SettingsType.EMAIL_CONFIGURATION, SmtpSettings.class);
if (!emailConfig.equals(STORED_SMTP_SETTINGS)) {
STORED_SMTP_SETTINGS = emailConfig;
SmtpSettings emailConfig = SettingsCache.getSetting(SettingsType.EMAIL_CONFIGURATION, SmtpSettings.class);
if (!emailConfig.equals(storedSmtpSettings)) {
storedSmtpSettings = emailConfig;
mailer = createMailer(emailConfig);
}
return emailConfig;

1
openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java
View File

@ -165,7 +165,6 @@ public final class EntityUtil {
}
List<EntityReference> refs = new ArrayList<>();
for (EntityRelationshipRecord ref : list) {
System.out.println("XXX getting entityRef for " + ref.getType() + " " + ref.getId());
refs.add(Entity.getEntityReferenceById(ref.getType(), ref.getId(), ALL));
}
refs.sort(compareEntityReference);

16
openmetadata-service/src/main/java/org/openmetadata/service/util/NotificationHandler.java
View File

@ -159,15 +159,13 @@ public class NotificationHandler {
id -> {
try {
User user = repository.get(null, id, repository.getFields("name,email,href"));
EmailUtil.getInstance()
.sendTaskAssignmentNotificationToUser(
user.getName(),
user.getEmail(),
String.format(
"%s/users/%s/tasks", EmailUtil.getInstance().buildBaseUrl(urlInstance), user.getName()),
thread,
EmailUtil.getInstance().getTaskAssignmentSubject(),
EmailUtil.TASK_NOTIFICATION_TEMPLATE);
EmailUtil.sendTaskAssignmentNotificationToUser(
user.getName(),
user.getEmail(),
String.format("%s/users/%s/tasks", EmailUtil.buildBaseUrl(urlInstance), user.getName()),
thread,
EmailUtil.getTaskAssignmentSubject(),
EmailUtil.TASK_NOTIFICATION_TEMPLATE);
} catch (IOException ex) {
LOG.error("Task Email Notification Failed :", ex);
} catch (TemplateException ex) {

8
openmetadata-service/src/main/java/org/openmetadata/service/util/SubscriptionUtil.java
View File

@ -113,19 +113,19 @@ public class SubscriptionUtil {
if (type == CreateEventSubscription.SubscriptionType.EMAIL
|| type == CreateEventSubscription.SubscriptionType.DATA_INSIGHT) {
if (USER.equals(owner.getType())) {
User user = SubjectCache.getInstance().getSubjectContext(owner.getId()).getUser();
User user = SubjectCache.getSubjectContext(owner.getId()).getUser();
data.add(user.getEmail());
} else {
Team team = SubjectCache.getInstance().getTeam(owner.getId());
Team team = SubjectCache.getTeam(owner.getId());
data.add(team.getEmail());
}
} else {
Profile profile = null;
if (USER.equals(owner.getType())) {
User user = SubjectCache.getInstance().getSubjectContext(owner.getId()).getUser();
User user = SubjectCache.getSubjectContext(owner.getId()).getUser();
profile = user.getProfile();
} else if (TEAM.equals(owner.getType())) {
Team team = SubjectCache.getInstance().getTeam(owner.getId());
Team team = SubjectCache.getTeam(owner.getId());
profile = team.getProfile();
}
data.addAll(getWebhookUrlsFromProfile(profile, owner.getId(), owner.getType(), type));

10
openmetadata-service/src/main/java/org/openmetadata/service/util/UserUtil.java
View File

@ -26,7 +26,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.api.configuration.pipelineServiceClient.PipelineServiceClientConfiguration;
import org.openmetadata.schema.auth.BasicAuthMechanism;
import org.openmetadata.schema.auth.JWTAuthMechanism;
import org.openmetadata.schema.auth.JWTTokenExpiry;
@ -37,7 +36,6 @@ import org.openmetadata.schema.services.connections.metadata.AuthProvider;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.utils.EntityInterfaceUtil;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.UserRepository;
@ -73,7 +71,7 @@ public final class UserUtil {
// Fetch Original User, is available
User originalUser =
userRepository.getByName(null, EntityInterfaceUtil.quoteName(username), new Fields(fieldList));
if (!originalUser.getIsBot() && !originalUser.getIsAdmin()) {
if (Boolean.FALSE.equals(originalUser.getIsBot()) && Boolean.FALSE.equals(originalUser.getIsAdmin())) {
updatedUser = originalUser;
// Update Auth Mechanism if not present, and send mail to the user
@ -115,7 +113,7 @@ public final class UserUtil {
private static String getPassword() {
try {
EmailUtil.getInstance().testConnection();
EmailUtil.testConnection();
return PasswordUtil.generateRandomPassword();
} catch (Exception ex) {
LOG.info("Password set to Default.");
@ -176,10 +174,8 @@ public final class UserUtil {
* </ul>
* </ul>
*/
public static User addOrUpdateBotUser(User user, OpenMetadataApplicationConfig openMetadataApplicationConfig) {
public static User addOrUpdateBotUser(User user) {
User originalUser = retrieveWithAuthMechanism(user);
PipelineServiceClientConfiguration pipelineServiceClientConfiguration =
openMetadataApplicationConfig.getPipelineServiceClientConfiguration();
AuthenticationMechanism authMechanism = originalUser != null ? originalUser.getAuthenticationMechanism() : null;
// the user did not have an auth mechanism and auth config is present
if (authMechanism == null) {

13
openmetadata-service/src/main/java/org/openmetadata/service/util/ValidatorUtil.java
View File

@ -5,10 +5,11 @@ import java.util.Arrays;
import java.util.Set;
import javax.validation.ConstraintViolation;
import javax.validation.Validation;
import javax.validation.Validator;
import javax.validation.ValidatorFactory;
public class ValidatorUtil {
public static final javax.validation.Validator VALIDATOR;
public static final Validator VALIDATOR;
static {
ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
@ -21,11 +22,9 @@ public class ValidatorUtil {
public static <T> String validate(T entity) {
Set<ConstraintViolation<T>> violations = VALIDATOR.validate(entity);
String ret =
violations.isEmpty()
? null
: Arrays.toString(
violations.stream().map(v -> String.format("%s %s", v.getPropertyPath(), v.getMessage())).toArray());
return ret;
return violations.isEmpty()
? null
: Arrays.toString(
violations.stream().map(v -> String.format("%s %s", v.getPropertyPath(), v.getMessage())).toArray());
}
}

12
openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
View File

@ -567,7 +567,6 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
Random rand = new Random();
int maxEntities = rand.nextInt(16) + 5;
System.out.println("XXX creating entities " + maxEntities);
List<UUID> createdUUIDs = new ArrayList<>();
for (int i = 0; i < maxEntities; i++) {
createdUUIDs.add(createEntity(createRequest(test, i + 1), ADMIN_AUTH_HEADERS).getId());
@ -580,8 +579,6 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
// Test listing entities that include deleted, non-deleted, and all the entities
Random random = new Random();
for (Include include : List.of(Include.NON_DELETED, Include.ALL, Include.DELETED)) {
System.out.println("XXX supportSoftDelete " + supportsSoftDelete);
System.out.println("XXX Include " + include);
if (!supportsSoftDelete && include.equals(Include.DELETED)) {
continue;
}
@ -592,7 +589,6 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
ResultList<T> allEntities = listEntities(queryParams, 1000000, null, null, ADMIN_AUTH_HEADERS);
int totalRecords = allEntities.getData().size();
printEntities(allEntities);
System.out.println("XXX totalRecords " + totalRecords);
// List entity with "limit" set from 1 to maxEntities size with random jumps (to reduce the test time)
// Each time compare the returned list with allTables list to make sure right results are returned
@ -605,8 +601,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
ResultList<T> backwardPage;
boolean foundDeleted = false;
do { // For each limit (or page size) - forward scroll till the end
LOG.info(
"XXX Limit {} forward pageCount {} indexInAllTables {} totalRecords {} afterCursor {}",
LOG.debug(
"Limit {} forward pageCount {} indexInAllTables {} totalRecords {} afterCursor {}",
limit,
pageCount,
indexInAllTables,
@ -2494,8 +2490,8 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
}
private void printEntities(ResultList<T> list) {
list.getData().forEach(e -> LOG.info("XXX {} {}", entityClass, e.getFullyQualifiedName()));
LOG.info("XXX before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
list.getData().forEach(e -> LOG.debug("{} {}", entityClass, e.getFullyQualifiedName()));
LOG.debug("before {} after {} ", list.getPaging().getBefore(), list.getPaging().getAfter());
}
public void assertEntityDeleted(T entity, boolean hardDelete) {

10
openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/RuleEvaluatorTest.java
View File

@ -44,8 +44,6 @@ class RuleEvaluatorTest {
Entity.registerEntity(User.class, Entity.USER, mock(UserRepository.class), null);
Entity.registerEntity(Team.class, Entity.TEAM, mock(TeamRepository.class), null);
Entity.registerEntity(Role.class, Entity.ROLE, mock(RoleRepository.class), null);
SubjectCache.initialize();
RoleCache.initialize();
TableRepository tableRepository = mock(TableRepository.class);
Mockito.when(tableRepository.getAllTags(any()))
@ -255,10 +253,10 @@ class RuleEvaluatorTest {
Team team = new Team().withName(teamName).withId(teamId);
if (parentName != null) {
UUID parentId = UUID.nameUUIDFromBytes(parentName.getBytes(StandardCharsets.UTF_8));
Team parentTeam = SubjectCache.getInstance().getTeam(parentId);
Team parentTeam = SubjectCache.getTeam(parentId);
team.setParents(listOf(parentTeam.getEntityReference()));
}
SubjectCache.teamCacheWithId.put(team.getId(), team);
SubjectCache.TEAM_CACHE_WITH_ID.put(team.getId(), team);
return team;
}
@ -268,7 +266,7 @@ class RuleEvaluatorTest {
team.setDefaultRoles(listOf(role.getEntityReference()));
team.setInheritedRoles(new ArrayList<>());
for (EntityReference parent : listOrEmpty(team.getParents())) {
Team parentTeam = SubjectCache.getInstance().getTeam(parent.getId());
Team parentTeam = SubjectCache.getTeam(parent.getId());
team.getInheritedRoles().addAll(listOrEmpty(parentTeam.getDefaultRoles()));
team.getInheritedRoles().addAll(listOrEmpty(parentTeam.getInheritedRoles()));
}
@ -278,7 +276,7 @@ class RuleEvaluatorTest {
private Role createRole(String roleName) {
UUID roleId = UUID.nameUUIDFromBytes(roleName.getBytes(StandardCharsets.UTF_8));
Role role = new Role().withName(roleName).withId(roleId);
RoleCache.roleCacheWithId.put(role.getId(), role);
RoleCache.CACHE_WITH_ID.put(role.getId(), role);
return role;
}

22
openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/SubjectContextTest.java
View File

@ -70,9 +70,6 @@ public class SubjectContextTest {
Entity.registerEntity(Team.class, Entity.TEAM, mock(TeamRepository.class), null);
Entity.registerEntity(Policy.class, Entity.POLICY, mock(PolicyRepository.class), null);
Entity.registerEntity(Role.class, Entity.ROLE, mock(RoleRepository.class), null);
PolicyCache.initialize();
RoleCache.initialize();
SubjectCache.initialize();
// Create team hierarchy:
// team1
@ -111,7 +108,7 @@ public class SubjectContextTest {
userRoles = getRoles("user");
List<EntityReference> userRolesRef = toEntityReferences(userRoles);
user = new User().withName("user").withRoles(userRolesRef).withTeams(List.of(team111.getEntityReference()));
SubjectCache.userCache.put("user", new SubjectContext(user));
SubjectCache.USER_CACHE.put("user", new SubjectContext(user));
}
@AfterAll
@ -124,7 +121,7 @@ public class SubjectContextTest {
@Test
void testPolicyIterator() {
// Check iteration order of the policies without resourceOwner
SubjectContext subjectContext = SubjectCache.getInstance().getSubjectContext(user.getName());
SubjectContext subjectContext = SubjectCache.getSubjectContext(user.getName());
Iterator<PolicyContext> policyContextIterator = subjectContext.getPolicies(null);
List<String> expectedUserPolicyOrder = new ArrayList<>();
expectedUserPolicyOrder.addAll(getPolicyListFromRoles(userRoles)); // First polices associated with user roles
@ -137,7 +134,7 @@ public class SubjectContextTest {
assertPolicyIterator(expectedUserPolicyOrder, policyContextIterator);
// Check iteration order of policies with team13 as the resource owner
subjectContext = SubjectCache.getInstance().getSubjectContext(user.getName());
subjectContext = SubjectCache.getSubjectContext(user.getName());
policyContextIterator = subjectContext.getPolicies(team13.getEntityReference());
List<String> expectedUserAndTeam13PolicyOrder = new ArrayList<>();
expectedUserAndTeam13PolicyOrder.addAll(expectedUserPolicyOrder);
@ -145,7 +142,7 @@ public class SubjectContextTest {
assertPolicyIterator(expectedUserAndTeam13PolicyOrder, policyContextIterator);
// Check iteration order of policies with team131 as the resource owner
subjectContext = SubjectCache.getInstance().getSubjectContext(user.getName());
subjectContext = SubjectCache.getSubjectContext(user.getName());
policyContextIterator = subjectContext.getPolicies(team131.getEntityReference());
// Roles & policies are inherited from resource owner team131
List<String> expectedUserAndTeam131PolicyOrder = new ArrayList<>();
@ -157,7 +154,7 @@ public class SubjectContextTest {
@Test
void testUserInHierarchy() {
SubjectContext subjectContext = SubjectCache.getInstance().getSubjectContext(user.getName());
SubjectContext subjectContext = SubjectCache.getSubjectContext(user.getName());
//
// Now test given user is part of team hierarchy
//
@ -171,7 +168,7 @@ public class SubjectContextTest {
@Test
void testResourceIsTeamAsset() {
SubjectContext subjectContext = SubjectCache.getInstance().getSubjectContext(user.getName());
SubjectContext subjectContext = SubjectCache.getSubjectContext(user.getName());
//
// Given entity owner "user", ensure isTeamAsset is true for teams 111, 11, 12, 1 and not for 13
@ -193,7 +190,6 @@ public class SubjectContextTest {
assertFalse(subjectContext.isTeamAsset("team13", teamOwner));
}
@Test
private static List<Role> getRoles(String prefix) {
// Create roles with 3 policies each and each policy with 3 rules
List<Role> roles = new ArrayList<>(3);
@ -201,7 +197,7 @@ public class SubjectContextTest {
String name = prefix + "_role_" + i;
List<EntityReference> policies = toEntityReferences(getPolicies(name));
Role role = new Role().withName(name).withId(UUID.randomUUID()).withPolicies(policies);
RoleCache.roleCacheWithId.put(role.getId(), role);
RoleCache.CACHE_WITH_ID.put(role.getId(), role);
roles.add(role);
}
return roles;
@ -213,7 +209,7 @@ public class SubjectContextTest {
String name = prefix + "_policy_" + i;
Policy policy = new Policy().withName(name).withId(UUID.randomUUID()).withRules(getRules(name));
policies.add(policy);
PolicyCache.policyCache.put(policy.getId(), PolicyCache.getInstance().getRules(policy));
PolicyCache.CACHE.put(policy.getId(), PolicyCache.getRules(policy));
}
return policies;
}
@ -268,7 +264,7 @@ public class SubjectContextTest {
.withDefaultRoles(toEntityReferences(roles))
.withPolicies(toEntityReferences(policies))
.withParents(parentList);
SubjectCache.teamCacheWithId.put(team.getId(), team);
SubjectCache.TEAM_CACHE_WITH_ID.put(team.getId(), team);
return team;
}