yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-15 12:37:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #8236 - Initialize bots from JSON data files (#8304)

Browse Source
This commit is contained in:
Suresh Srinivas 2022-10-21 08:38:52 -07:00 committed by GitHub
parent 339f123f92
commit 229b56e7b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 177 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bootstrap/sql/com.mysql.cj.jdbc.Driver/v006__create_db_connection_info.sql
View File

@ -9,4 +9,7 @@ CREATE TABLE IF NOT EXISTS web_analytic_event (
deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
UNIQUE(name),
INDEX name_index (name)
);
);
UPDATE bot_entity
SET json = JSON_INSERT(JSON_REMOVE(json, '$.botType'), '$.provider', 'system');

6
bootstrap/sql/org.postgresql.Driver/v006__create_db_connection_info.sql
View File

@ -11,3 +11,9 @@ CREATE TABLE IF NOT EXISTS web_analytic_event (
);
CREATE INDEX IF NOT EXISTS name_index ON web_analytic_event(name);
UPDATE bot_entity
SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true);
UPDATE bot_entity
SET json = json::jsonb #- '{botType}';

3
ingestion/src/metadata/ingestion/ometa/ometa_api.py
View File

@ -26,7 +26,6 @@ from pydantic import BaseModel
from requests.utils import quote
from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
from metadata.generated.schema.entity.bot import BotType
from metadata.generated.schema.entity.data.chart import Chart
from metadata.generated.schema.entity.data.dashboard import Dashboard
from metadata.generated.schema.entity.data.database import Database
@ -180,7 +179,7 @@ class OpenMetadata(
# Load auth provider config from Secret Manager if necessary
self.secrets_manager_client.add_auth_provider_security_config(
self.config, BotType.ingestion_bot.value
self.config, "ingestion-bot"
)
# Load the auth provider init from the registry

5
ingestion/tests/unit/metadata/utils/secrets/test_aws_based_secrets_manager.py
View File

@ -17,7 +17,6 @@ from copy import deepcopy
from typing import Any, Dict
from unittest.mock import Mock, patch
from metadata.generated.schema.entity.bot import BotType
from metadata.generated.schema.entity.services.connections.serviceConnection import (
ServiceConnection,
)
@ -88,7 +87,7 @@ class AWSBasedSecretsManager(object):
actual_om_connection.securityConfig = None
aws_manager.add_auth_provider_security_config(
actual_om_connection, BotType.ingestion_bot.value
actual_om_connection, "ingestion-bot"
)
self.assert_client_called_once(
@ -111,7 +110,7 @@ class AWSBasedSecretsManager(object):
with self.assertRaises(ValueError) as value_error:
aws_manager.add_auth_provider_security_config(
self.om_connection, BotType.ingestion_bot.value
self.om_connection, "ingestion-bot"
)
self.assertTrue(
"/openmetadata/bot/ingestion-bot" in str(value_error.exception)

3
ingestion/tests/unit/metadata/utils/secrets/test_noop_secrets_manager.py
View File

@ -14,7 +14,6 @@ Test Local Secrets Manager
"""
from copy import deepcopy
from metadata.generated.schema.entity.bot import BotType
from metadata.generated.schema.entity.services.connections.metadata.secretsManagerProvider import (
SecretsManagerProvider,
)
@ -57,7 +56,7 @@ class TestLocalSecretsManager(TestSecretsManager.External):
actual_om_connection.securityConfig = self.auth_provider_config
noop_manager.add_auth_provider_security_config(
actual_om_connection, BotType.ingestion_bot.value
actual_om_connection, "ingestion-bot"
)
self.assertEqual(self.auth_provider_config, actual_om_connection.securityConfig)

2
openmetadata-service/src/main/java/org/openmetadata/service/Entity.java
View File

@ -130,7 +130,7 @@ public final class Entity {
//
public static final String ADMIN_USER_NAME = "admin";
public static final String ORGANIZATION_NAME = "Organization";
public static final String DATACONSUMER_ROLE = "DataConsumer";
public static final String INGESTION_BOT_NAME = "ingestion-bot";
//
// List of entities whose changes should not be published to the Activity Feed

4
openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java
View File

@ -175,7 +175,7 @@ public final class CatalogExceptionMessage {
return String.format("Failed to evaluate - %s", message);
}
public static String deletionNotAllowed(String entityType, String name) {
return String.format("Deletion of %s %s is not allowed", entityType, name);
public static String systemEntityDeleteNotAllowed(String name, String entityType) {
return String.format("System entity [%s] of type %s can not be deleted.", name, entityType);
}
}

18
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/BotRepository.java
View File

@ -15,12 +15,13 @@ package org.openmetadata.service.jdbi3;
import java.io.IOException;
import org.openmetadata.schema.entity.Bot;
import org.openmetadata.schema.entity.BotType;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.ProviderType;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.CatalogExceptionMessage;
import org.openmetadata.service.resources.bots.BotResource;
import org.openmetadata.service.secrets.SecretsManager;
import org.openmetadata.service.secrets.SecretsManagerFactory;
@ -51,9 +52,9 @@ public class BotRepository extends EntityRepository<Bot> {
EntityReference botUser = entity.getBotUser();
entity.withBotUser(null);
store(entity.getId(), entity, update);
if (!BotType.BOT.equals(entity.getBotType())) {
if (!ProviderType.USER.equals(entity.getProvider())) { // encryption is done only for system bots
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
secretsManager.encryptOrDecryptBotCredentials(entity.getBotType().value(), botUser.getName(), true);
secretsManager.encryptOrDecryptBotCredentials(entity.getName(), botUser.getName(), true);
}
entity.withBotUser(botUser);
}
@ -68,6 +69,14 @@ public class BotRepository extends EntityRepository<Bot> {
return new BotUpdater(original, updated, operation);
}
@Override
protected void preDelete(Bot entity) {
if (entity.getProvider() == ProviderType.SYSTEM) { // System provided bot can't be deleted
throw new IllegalArgumentException(
CatalogExceptionMessage.systemEntityDeleteNotAllowed(entity.getName(), Entity.BOT));
}
}
@Override
public void restorePatchAttributes(Bot original, Bot updated) {
// Bot user can't be changed by patch
@ -86,9 +95,6 @@ public class BotRepository extends EntityRepository<Bot> {
@Override
public void entitySpecificUpdate() throws IOException {
updateUser(original, updated);
if (original.getBotType() != null) {
updated.setBotType(original.getBotType());
}
}
private void updateUser(Bot original, Bot updated) throws IOException {

6
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
View File

@ -254,16 +254,16 @@ public abstract class EntityRepository<T extends EntityInterface> {
}
public List<T> getEntitiesFromSeedData(String path) throws IOException {
return getEntitiesFromSeedData(path, entityClass);
return getEntitiesFromSeedData(entityType, path, entityClass);
}
public <U> List<U> getEntitiesFromSeedData(String path, Class<U> clazz) throws IOException {
public static <U> List<U> getEntitiesFromSeedData(String entityType, String path, Class<U> clazz) throws IOException {
List<U> entities = new ArrayList<>();
List<String> jsonDataFiles = EntityUtil.getJsonDataResources(path);
jsonDataFiles.forEach(
jsonDataFile -> {
try {
String json = CommonUtil.getResourceAsStream(getClass().getClassLoader(), jsonDataFile);
String json = CommonUtil.getResourceAsStream(EntityRepository.class.getClassLoader(), jsonDataFile);
json = json.replace("<separator>", Entity.SEPARATOR);
entities.add(JsonUtils.readValue(json, clazz));
} catch (Exception e) {

3
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PolicyRepository.java
View File

@ -143,7 +143,8 @@ public class PolicyRepository extends EntityRepository<Policy> {
@Override
protected void preDelete(Policy entity) {
if (FALSE.equals(entity.getAllowDelete())) {
throw new IllegalArgumentException(CatalogExceptionMessage.deletionNotAllowed(Entity.POLICY, entity.getName()));
throw new IllegalArgumentException(
CatalogExceptionMessage.systemEntityDeleteNotAllowed(entity.getName(), Entity.POLICY));
}
}

3
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/RoleRepository.java
View File

@ -114,7 +114,8 @@ public class RoleRepository extends EntityRepository<Role> {
@Override
protected void preDelete(Role entity) {
if (FALSE.equals(entity.getAllowDelete())) {
throw new IllegalArgumentException(CatalogExceptionMessage.deletionNotAllowed(Entity.ROLE, entity.getName()));
throw new IllegalArgumentException(
CatalogExceptionMessage.systemEntityDeleteNotAllowed(entity.getName(), Entity.ROLE));
}
}

2
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java
View File

@ -93,7 +93,7 @@ public class UserRepository extends EntityRepository<User> {
private List<EntityReference> getInheritedRoles(User user) throws IOException {
getTeams(user);
return SubjectCache.getInstance().getRolesForTeams(getTeams(user));
return SubjectCache.getInstance() != null ? SubjectCache.getInstance().getRolesForTeams(getTeams(user)) : null;
}
@Override

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/Collection.java
View File

@ -23,6 +23,6 @@ import java.lang.annotation.Target;
public @interface Collection {
String name();
/** Only order from 0 to 9 (inclusive) are allowed */
/** Order of initialization of resource starting from 0. Only order from 0 to 9 (inclusive) are allowed */
int order() default 9;
}

4
openmetadata-service/src/main/java/org/openmetadata/service/resources/CollectionRegistry.java
View File

@ -264,8 +264,10 @@ public final class CollectionRegistry {
try {
Method initializeMethod = resource.getClass().getMethod("initialize", OpenMetadataApplicationConfig.class);
initializeMethod.invoke(resource, config);
} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ignored) {
} catch (NoSuchMethodException ignored) {
// Method does not exist and initialize is not called
} catch (Exception ex) {
LOG.warn("Encountered exception ", ex);
}
return resource;
}

37
openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotResource.java
View File

@ -13,6 +13,8 @@
package org.openmetadata.service.resources.bots;
import static org.openmetadata.service.security.DefaultAuthorizer.user;
import io.swagger.annotations.Api;
import io.swagger.v3.oas.annotations.ExternalDocumentation;
import io.swagger.v3.oas.annotations.Operation;
@ -45,14 +47,16 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.SecurityContext;
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.api.CreateBot;
import org.openmetadata.schema.entity.Bot;
import org.openmetadata.schema.entity.BotType;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.type.EntityHistory;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.ProviderType;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.jdbi3.BotRepository;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.ListFilter;
@ -62,9 +66,12 @@ import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.secrets.SecretsManager;
import org.openmetadata.service.secrets.SecretsManagerFactory;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.security.DefaultAuthorizer;
import org.openmetadata.service.security.SecurityUtil;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.ResultList;
@Slf4j
@Path("/v1/bots")
@Api(value = "Bot collection", tags = "Bot collection")
@Produces(MediaType.APPLICATION_JSON)
@ -77,6 +84,23 @@ public class BotResource extends EntityResource<Bot, BotRepository> {
super(Bot.class, new BotRepository(dao), authorizer);
}
@SuppressWarnings("unused") // Method is used by reflection
public void initialize(OpenMetadataApplicationConfig config) throws IOException {
// Load system bots
List<Bot> bots = dao.getEntitiesFromSeedData();
String domain = SecurityUtil.getDomain(config);
for (Bot bot : bots) {
String userName = bot.getBotUser().getName();
User user = user(userName, domain, userName).withIsBot(true).withIsAdmin(false);
user = DefaultAuthorizer.addOrUpdateBotUser(user, config);
bot.withId(UUID.randomUUID())
.withBotUser(user.getEntityReference())
.withUpdatedBy(userName)
.withUpdatedAt(System.currentTimeMillis());
dao.initializeEntity(bot);
}
}
@Override
public Bot addHref(UriInfo uriInfo, Bot entity) {
Entity.withHref(uriInfo, entity.getBotUser());
@ -265,10 +289,11 @@ public class BotResource extends EntityResource<Bot, BotRepository> {
Bot bot = getBot(securityContext, create);
Response response = createOrUpdate(uriInfo, securityContext, bot, false);
// ensures the secrets' manager store the credentials even when the botUser does not change
// TODO encrypt decrypt could be done twice
bot = (Bot) response.getEntity();
SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
if (!BotType.BOT.equals(bot.getBotType())) {
secretsManager.encryptOrDecryptBotCredentials(bot.getBotType().value(), bot.getBotUser().getName(), true);
if (!ProviderType.USER.equals(bot.getProvider())) { // User bots credentials are not stored. Only system bots.
secretsManager.encryptOrDecryptBotCredentials(bot.getName(), bot.getBotUser().getName(), true);
}
return response;
}
@ -319,17 +344,13 @@ public class BotResource extends EntityResource<Bot, BotRepository> {
boolean hardDelete,
@Parameter(description = "Id of the Bot", schema = @Schema(type = "UUID")) @PathParam("id") UUID id)
throws IOException {
BotType botType = dao.get(null, id, EntityUtil.Fields.EMPTY_FIELDS).getBotType();
if (!BotType.BOT.equals(botType)) {
throw new IllegalArgumentException(String.format("[%s] can not be deleted.", botType.value()));
}
return delete(uriInfo, securityContext, id, true, hardDelete, false);
}
private Bot getBot(CreateBot create, String user) throws IOException {
return copy(new Bot(), create, user)
.withBotUser(create.getBotUser())
.withBotType(BotType.BOT)
.withProvider(create.getProvider())
.withFullyQualifiedName(create.getName());
}

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java
View File

@ -95,7 +95,7 @@ public class PolicyResource extends EntityResource<Policy, PolicyRepository> {
super(Policy.class, new PolicyRepository(dao), authorizer);
}
@SuppressWarnings("unused") // Method is used for reflection
@SuppressWarnings("unused") // Method is used by reflection
public void initialize(OpenMetadataApplicationConfig config) throws IOException {
// Load any existing rules from database, before loading seed data.
dao.initSeedDataFromResources();

3
openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java
View File

@ -93,7 +93,8 @@ public class TagResource {
@SuppressWarnings("unused") // Method used by reflection
public void initialize(OpenMetadataApplicationConfig config) throws IOException {
// Find tag definitions and load tag categories from the json file, if necessary
List<TagCategory> tagCategories = dao.getEntitiesFromSeedData(".*json/data/tags/.*\\.json$", TagCategory.class);
List<TagCategory> tagCategories =
dao.getEntitiesFromSeedData(Entity.TAG_CATEGORY, ".*json/data/tags/.*\\.json$", TagCategory.class);
for (TagCategory tagCategory : tagCategories) {
long now = System.currentTimeMillis();
tagCategory.withId(UUID.randomUUID()).withUpdatedBy(ADMIN_USER_NAME).withUpdatedAt(now);

2
openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java
View File

@ -141,7 +141,7 @@ import org.openmetadata.service.util.TokenUtil;
@Api(value = "User collection", tags = "User collection")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
@Collection(name = "users")
@Collection(name = "users", order = 8) // Initialize user resource before bot resource (at default order 9)
public class UserResource extends EntityResource<User, UserRepository> {
public static final String COLLECTION_PATH = "v1/users/";
public static final String USER_PROTECTED_FIELDS = "authenticationMechanism";

151
openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java
View File

@ -23,27 +23,21 @@ import static org.openmetadata.schema.teams.authn.SSOAuthMechanism.SsoServiceTyp
import static org.openmetadata.service.Entity.ADMIN_USER_NAME;
import static org.openmetadata.service.exception.CatalogExceptionMessage.notAdmin;
import static org.openmetadata.service.resources.teams.UserResource.USER_PROTECTED_FIELDS;
import static org.openmetadata.service.security.SecurityUtil.DEFAULT_PRINCIPAL_DOMAIN;
import at.favre.lib.crypto.bcrypt.BCrypt;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.ws.rs.core.SecurityContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.jdbi.v3.core.Jdbi;
import org.openmetadata.schema.api.configuration.airflow.AirflowConfiguration;
import org.openmetadata.schema.api.security.AuthenticationConfiguration;
import org.openmetadata.schema.email.SmtpSettings;
import org.openmetadata.schema.entity.Bot;
import org.openmetadata.schema.entity.BotType;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.security.client.OpenMetadataJWTClientConfig;
@ -57,7 +51,6 @@ import org.openmetadata.schema.type.ResourcePermission;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.exception.EntityNotFoundException;
import org.openmetadata.service.jdbi3.BotRepository;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.jdbi3.UserRepository;
import org.openmetadata.service.secrets.SecretsManager;
@ -78,88 +71,52 @@ import org.openmetadata.service.util.RestUtil;
@Slf4j
public class DefaultAuthorizer implements Authorizer {
private static final String COLON_DELIMITER = ":";
private static final String DEFAULT_ADMIN = ADMIN_USER_NAME;
private Set<String> adminUsers;
private Set<String> botPrincipalUsers;
private Set<String> testUsers;
private String principalDomain;
private String providerType;
private boolean isSmtpEnabled;
@Override
public void init(OpenMetadataApplicationConfig openMetadataApplicationConfig, Jdbi dbi) {
LOG.info(
"Initializing DefaultAuthorizer with config {}", openMetadataApplicationConfig.getAuthorizerConfiguration());
this.adminUsers = new HashSet<>(openMetadataApplicationConfig.getAuthorizerConfiguration().getAdminPrincipals());
this.botPrincipalUsers =
new HashSet<>(openMetadataApplicationConfig.getAuthorizerConfiguration().getBotPrincipals());
this.testUsers = new HashSet<>(openMetadataApplicationConfig.getAuthorizerConfiguration().getTestPrincipals());
this.principalDomain = openMetadataApplicationConfig.getAuthorizerConfiguration().getPrincipalDomain();
this.providerType = openMetadataApplicationConfig.getAuthenticationConfiguration().getProvider();
SmtpSettings smtpSettings = openMetadataApplicationConfig.getSmtpSettings();
public void init(OpenMetadataApplicationConfig config, Jdbi dbi) {
LOG.info("Initializing DefaultAuthorizer with config {}", config.getAuthorizerConfiguration());
SmtpSettings smtpSettings = config.getSmtpSettings();
this.isSmtpEnabled = smtpSettings != null && smtpSettings.getEnableSmtpServer();
SubjectCache.initialize();
PolicyCache.initialize();
RoleCache.initialize();
LOG.debug("Admin users: {}", adminUsers);
initializeUsers(openMetadataApplicationConfig);
initializeUsers(config);
}
private void initializeUsers(OpenMetadataApplicationConfig openMetadataApplicationConfig) {
LOG.debug("Checking user entries for admin users");
String domain = principalDomain.isEmpty() ? DEFAULT_PRINCIPAL_DOMAIN : principalDomain;
if (!providerType.equals(SSOAuthMechanism.SsoServiceType.BASIC.value())) {
for (String adminUser : adminUsers) {
User user = user(adminUser, domain, adminUser).withIsAdmin(true);
addOrUpdateUser(user);
}
private void initializeUsers(OpenMetadataApplicationConfig config) {
Set<String> adminUsers = new HashSet<>(config.getAuthorizerConfiguration().getAdminPrincipals());
LOG.debug("Checking user entries for admin users {}", adminUsers);
String domain = SecurityUtil.getDomain(config);
String providerType = config.getAuthenticationConfiguration().getProvider();
if (providerType.equals(SSOAuthMechanism.SsoServiceType.BASIC.value())) {
handleBasicAuth(adminUsers, domain);
} else {
try {
handleBasicAuth(adminUsers, domain);
} catch (IOException e) {
LOG.error("Failed in Basic Auth Setup. Reason : {}", e.getMessage());
}
}
LOG.debug("Checking user entries for bot users");
Set<String> botUsers = Arrays.stream(BotType.values()).map(BotType::value).collect(Collectors.toSet());
botUsers.remove(BotType.BOT.value());
botUsers.addAll(botPrincipalUsers);
for (String botUser : botUsers) {
User user = user(botUser, domain, botUser).withIsBot(true).withIsAdmin(false);
user = addOrUpdateBotUser(user, openMetadataApplicationConfig);
if (user != null) {
BotType botType;
try {
botType = BotType.fromValue(botUser);
} catch (IllegalArgumentException e) {
botType = BotType.BOT;
}
Bot bot = bot(user).withBotUser(user.getEntityReference()).withBotType(botType);
addOrUpdateBot(bot);
}
addUsers(adminUsers, domain, true);
}
LOG.debug("Checking user entries for test users");
for (String testUser : testUsers) {
User user = user(testUser, domain, testUser);
addOrUpdateUser(user);
}
Set<String> testUsers = new HashSet<>(config.getAuthorizerConfiguration().getTestPrincipals());
addUsers(testUsers, domain, null);
}
private void handleBasicAuth(Set<String> adminUsers, String domain) throws IOException {
for (String adminUser : adminUsers) {
if (adminUser.contains(COLON_DELIMITER)) {
String[] tokens = adminUser.split(COLON_DELIMITER);
addUserForBasicAuth(tokens[0], tokens[1], domain);
} else {
boolean isDefaultAdmin = adminUser.equals(DEFAULT_ADMIN);
String token = PasswordUtil.generateRandomPassword();
if (isDefaultAdmin || !isSmtpEnabled) {
token = DEFAULT_ADMIN;
private void handleBasicAuth(Set<String> adminUsers, String domain) {
try {
for (String adminUser : adminUsers) {
if (adminUser.contains(COLON_DELIMITER)) {
String[] tokens = adminUser.split(COLON_DELIMITER);
addUserForBasicAuth(tokens[0], tokens[1], domain);
} else {
boolean isDefaultAdmin = adminUser.equals(ADMIN_USER_NAME);
String token = PasswordUtil.generateRandomPassword();
if (isDefaultAdmin || !isSmtpEnabled) {
token = ADMIN_USER_NAME;
}
addUserForBasicAuth(adminUser, token, domain);
}
addUserForBasicAuth(adminUser, token, domain);
}
} catch (IOException e) {
LOG.error("Failed in Basic Auth Setup. Reason : {}", e.getMessage());
}
}
@ -213,7 +170,7 @@ public class DefaultAuthorizer implements Authorizer {
.withConfig(new BasicAuthMechanism().withPassword(hashedPwd)));
}
private User user(String name, String domain, String updatedBy) {
public static User user(String name, String domain, String updatedBy) {
return new User()
.withId(UUID.randomUUID())
.withName(name)
@ -224,16 +181,6 @@ public class DefaultAuthorizer implements Authorizer {
.withIsBot(false);
}
private Bot bot(User user) {
return new Bot()
.withId(UUID.randomUUID())
.withName(user.getName())
.withFullyQualifiedName(user.getName())
.withUpdatedBy(user.getUpdatedBy())
.withUpdatedAt(System.currentTimeMillis())
.withDisplayName(user.getName());
}
@Override
public List<ResourcePermission> listPermissions(SecurityContext securityContext, String user) {
SubjectContext subjectContext = getSubjectContext(securityContext);
@ -307,7 +254,14 @@ public class DefaultAuthorizer implements Authorizer {
throw new AuthorizationException(notAdmin(securityContext.getUserPrincipal().getName()));
}
private User addOrUpdateUser(User user) {
private void addUsers(Set<String> users, String domain, Boolean isAdmin) {
for (String userName : users) {
User user = user(userName, domain, userName).withIsAdmin(isAdmin);
addOrUpdateUser(user);
}
}
private static User addOrUpdateUser(User user) {
EntityRepository<User> userRepository = Entity.getEntityRepository(Entity.USER);
try {
RestUtil.PutResponse<User> addedUser = userRepository.createOrUpdate(null, user);
@ -316,7 +270,7 @@ public class DefaultAuthorizer implements Authorizer {
return addedUser.getEntity();
} catch (Exception exception) {
// In HA set up the other server may have already added the user.
LOG.debug("Caught exception: {}", ExceptionUtils.getStackTrace(exception));
LOG.debug("Caught exception", exception);
user.setAuthenticationMechanism(null);
LOG.debug("User entry: {} already exists.", user.getName());
}
@ -341,7 +295,7 @@ public class DefaultAuthorizer implements Authorizer {
* </ul>
* </ul>
*/
private User addOrUpdateBotUser(User user, OpenMetadataApplicationConfig openMetadataApplicationConfig) {
public static User addOrUpdateBotUser(User user, OpenMetadataApplicationConfig openMetadataApplicationConfig) {
User originalUser = retrieveAuthMechanism(user);
// the user did not have an auth mechanism
AuthenticationMechanism authMechanism = originalUser != null ? originalUser.getAuthenticationMechanism() : null;
@ -402,15 +356,16 @@ public class DefaultAuthorizer implements Authorizer {
return addOrUpdateUser(user);
}
private SSOAuthMechanism buildAuthMechanismConfig(SSOAuthMechanism.SsoServiceType ssoServiceType, Object config) {
private static SSOAuthMechanism buildAuthMechanismConfig(
SSOAuthMechanism.SsoServiceType ssoServiceType, Object config) {
return new SSOAuthMechanism().withSsoServiceType(ssoServiceType).withAuthConfig(config);
}
private AuthenticationMechanism buildAuthMechanism(AuthenticationMechanism.AuthType authType, Object config) {
private static AuthenticationMechanism buildAuthMechanism(AuthenticationMechanism.AuthType authType, Object config) {
return new AuthenticationMechanism().withAuthType(authType).withConfig(config);
}
private User retrieveAuthMechanism(User user) {
private static User retrieveAuthMechanism(User user) {
EntityRepository<User> userRepository = UserRepository.class.cast(Entity.getEntityRepository(Entity.USER));
try {
User originalUser =
@ -429,24 +384,6 @@ public class DefaultAuthorizer implements Authorizer {
}
}
private void addOrUpdateBot(Bot bot) {
EntityRepository<Bot> botRepository = BotRepository.class.cast(Entity.getEntityRepository(Entity.BOT));
Bot originalBot;
try {
originalBot = botRepository.getByName(null, bot.getName(), EntityUtil.Fields.EMPTY_FIELDS);
bot.setBotUser(originalBot.getBotUser());
} catch (Exception e) {
}
try {
RestUtil.PutResponse<Bot> addedBot = botRepository.createOrUpdate(null, bot);
LOG.debug("Added bot entry: {}", addedBot.getEntity().getName());
} catch (Exception exception) {
// In HA set up the other server may have already added the bot.
LOG.debug("Caught exception: {}", ExceptionUtils.getStackTrace(exception));
LOG.debug("Bot entry: {} already exists.", bot.getName());
}
}
private SubjectContext getSubjectContext(SecurityContext securityContext) {
if (securityContext == null || securityContext.getUserPrincipal() == null) {
throw new AuthenticationException("No principal in security context");

3
openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java
View File

@ -18,7 +18,6 @@ import java.util.List;
import java.util.UUID;
import javax.ws.rs.core.SecurityContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.jdbi.v3.core.Jdbi;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.type.EntityReference;
@ -99,7 +98,7 @@ public class NoopAuthorizer implements Authorizer {
LOG.debug("Added anonymous user entry: {}", addedUser);
} catch (IOException exception) {
// In HA set up the other server may have already added the user.
LOG.debug("Caught exception: {}", ExceptionUtils.getStackTrace(exception));
LOG.debug("Caught exception ", exception);
LOG.debug("Anonymous user entry: {} already exists.", user);
}
}

7
openmetadata-service/src/main/java/org/openmetadata/service/security/SecurityUtil.java
View File

@ -19,6 +19,8 @@ import java.security.Principal;
import java.util.Map;
import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.service.OpenMetadataApplicationConfig;
public final class SecurityUtil {
public static final String DEFAULT_PRINCIPAL_DOMAIN = "openmetadata.org";
@ -46,6 +48,11 @@ public final class SecurityUtil {
return principal == null ? null : principal.split("@")[0];
}
public static String getDomain(OpenMetadataApplicationConfig config) {
String principalDomain = config.getAuthorizerConfiguration().getPrincipalDomain();
return CommonUtil.nullOrEmpty(principalDomain) ? DEFAULT_PRINCIPAL_DOMAIN : principalDomain;
}
public static Invocation.Builder addHeaders(WebTarget target, Map<String, String> headers) {
if (headers != null) {
return target

3
openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataServerConnectionBuilder.java
View File

@ -6,7 +6,6 @@ import lombok.extern.slf4j.Slf4j;
import org.openmetadata.api.configuration.airflow.SSLConfig;
import org.openmetadata.schema.api.configuration.airflow.AirflowConfiguration;
import org.openmetadata.schema.entity.Bot;
import org.openmetadata.schema.entity.BotType;
import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.security.client.OpenMetadataJWTClientConfig;
@ -109,7 +108,7 @@ public class OpenMetadataServerConnectionBuilder {
}
private User retrieveBotUser() {
User botUser = retrieveIngestionBotUser(BotType.INGESTION_BOT.value());
User botUser = retrieveIngestionBotUser(Entity.INGESTION_BOT_NAME);
if (botUser == null) {
throw new IllegalArgumentException("Please, verify that the ingestion-bot is present.");
}

9
openmetadata-service/src/main/java/org/openmetadata/service/util/TablesInitializer.java
View File

@ -50,6 +50,7 @@ import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.teams.authn.GenerateTokenRequest;
import org.openmetadata.schema.teams.authn.JWTAuthMechanism;
import org.openmetadata.schema.teams.authn.JWTTokenExpiry;
import org.openmetadata.service.Entity;
import org.openmetadata.service.OpenMetadataApplicationConfig;
import org.openmetadata.service.elasticsearch.ElasticSearchIndexDefinition;
import org.openmetadata.service.fernet.Fernet;
@ -57,6 +58,7 @@ import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.UserRepository;
import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator;
import org.openmetadata.service.secrets.SecretsManagerFactory;
import org.openmetadata.service.security.SecurityUtil;
import org.openmetadata.service.security.jwt.JWTTokenGenerator;
public final class TablesInitializer {
@ -333,11 +335,8 @@ public final class TablesInitializer {
}
private static void createIngestionBot(OpenMetadataApplicationConfig config, Jdbi jdbi) {
String domain =
config.getAuthorizerConfiguration().getPrincipalDomain().isEmpty()
? DEFAULT_PRINCIPAL_DOMAIN
: config.getAuthorizerConfiguration().getPrincipalDomain();
String botUser = "ingestion-bot";
String domain = SecurityUtil.getDomain(config);
String botUser = Entity.INGESTION_BOT_NAME;
User user =
new User()

10
openmetadata-service/src/main/resources/json/data/bot/ingestionBot.json Normal file
View File

@ -0,0 +1,10 @@
{
"name": "ingestion-bot",
"description": "Bot used for ingesting metadata.",
"fullyQualifiedName": "ingestion-bot",
"botUser": {
"name" : "ingestion-bot",
"type" : "user"
},
"provider": "system"
}

20
openmetadata-service/src/test/java/org/openmetadata/service/resources/bots/BotResourceTest.java
View File

@ -1,11 +1,13 @@
package org.openmetadata.service.resources.bots;
import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
import static org.openmetadata.service.util.TestUtils.assertResponse;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import lombok.SneakyThrows;
@ -16,10 +18,11 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.openmetadata.schema.api.CreateBot;
import org.openmetadata.schema.entity.Bot;
import org.openmetadata.schema.entity.BotType;
import org.openmetadata.schema.entity.teams.User;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.service.Entity;
import org.openmetadata.service.exception.CatalogExceptionMessage;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.resources.EntityResourceTest;
import org.openmetadata.service.resources.bots.BotResource.BotList;
import org.openmetadata.service.resources.teams.UserResourceTest;
@ -52,6 +55,15 @@ public class BotResourceTest extends EntityResourceTest<Bot, CreateBot> {
}
}
@Test
void testBotInitialization() throws IOException {
// Ensure all the bots are bootstrapped from the data files
List<Bot> bots = EntityRepository.getEntitiesFromSeedData(Entity.BOT, ".*json/data/bot/.*\\.json$", Bot.class);
for (Bot bot : bots) {
assertNotNull(getEntityByName(bot.getName(), "", ADMIN_AUTH_HEADERS));
}
}
@Test
void put_entityNonEmptyDescriptionUpdate_200(TestInfo test) {
// PUT based updates are categorized as create operation
@ -103,14 +115,14 @@ public class BotResourceTest extends EntityResourceTest<Bot, CreateBot> {
}
@Test
void delete_failIfUserIsIngestionBot(TestInfo test) throws IOException {
void delete_failIfUserIsIngestionBot() throws IOException {
// get ingestion bot
Bot ingestionBot = getEntityByName(BotType.INGESTION_BOT.value(), "", ADMIN_AUTH_HEADERS);
Bot ingestionBot = getEntityByName(Entity.INGESTION_BOT_NAME, "", ADMIN_AUTH_HEADERS);
// fail because it is a system bot
assertResponse(
() -> deleteEntity(ingestionBot.getId(), true, true, ADMIN_AUTH_HEADERS),
BAD_REQUEST,
"[ingestion-bot] can not be deleted.");
CatalogExceptionMessage.systemEntityDeleteNotAllowed(ingestionBot.getName(), Entity.BOT));
}
@Override

2
openmetadata-service/src/test/java/org/openmetadata/service/resources/policies/PolicyResourceTest.java
View File

@ -243,7 +243,7 @@ public class PolicyResourceTest extends EntityResourceTest<Policy, CreatePolicy>
assertResponse(
() -> deleteEntity(policy.getId(), ADMIN_AUTH_HEADERS),
BAD_REQUEST,
CatalogExceptionMessage.deletionNotAllowed(Entity.POLICY, policy.getName()));
CatalogExceptionMessage.systemEntityDeleteNotAllowed(policy.getName(), Entity.POLICY));
}
}

2
openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/RoleResourceTest.java
View File

@ -142,7 +142,7 @@ public class RoleResourceTest extends EntityResourceTest<Role, CreateRole> {
assertResponse(
() -> deleteEntity(role.getId(), ADMIN_AUTH_HEADERS),
BAD_REQUEST,
CatalogExceptionMessage.deletionNotAllowed(Entity.ROLE, role.getName()));
CatalogExceptionMessage.systemEntityDeleteNotAllowed(role.getName(), Entity.ROLE));
}
}

4
openmetadata-spec/src/main/resources/json/schema/api/createBot.json
View File

@ -23,6 +23,10 @@
"description": {
"description": "Description of the bot.",
"type": "string"
},
"provider" : {
"$ref": "../type/basic.json#/definitions/providerType",
"default": "user"
}
},
"required": ["name", "botUser"],

15
openmetadata-spec/src/main/resources/json/schema/entity/bot.json
View File

@ -6,15 +6,6 @@
"type": "object",
"javaType": "org.openmetadata.schema.entity.Bot",
"javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
"definitions": {
"botType": {
"javaType": "org.openmetadata.schema.entity.BotType",
"description": "Type of bot",
"type": "string",
"enum": ["bot", "ingestion-bot"],
"default": "bot"
}
},
"properties": {
"id": {
"description": "Unique identifier of a bot instance.",
@ -40,9 +31,9 @@
"description": "Bot user created for this bot on behalf of which the bot performs all the operations, such as updating description, responding on the conversation threads, etc.",
"$ref" : "../type/entityReference.json"
},
"botType" : {
"$ref": "#/definitions/botType",
"default": "bot"
"provider" : {
"$ref": "../type/basic.json#/definitions/providerType",
"default": "user"
},
"version": {
"description": "Metadata version of the entity.",

7
openmetadata-spec/src/main/resources/json/schema/type/basic.json
View File

@ -120,6 +120,13 @@
},
"entityExtension" : {
"description": "Entity extension data with custom attributes added to the entity."
},
"providerType": {
"javaType": "org.openmetadata.schema.type.ProviderType",
"description": "Type of provider of an entity. Some entities are provided by the `system`. Some are are entities created and provided by the `user`. Typically `system` provide entities can't be deleted and can only be disabled.",
"type": "string",
"enum": ["system", "user"],
"default": "user"
}
}
}

8
openmetadata-ui/src/main/resources/ui/src/components/BotListV1/BotListV1.component.tsx
View File

@ -29,7 +29,7 @@ import {
NO_PERMISSION_FOR_ACTION,
} from '../../constants/HelperTextUtil';
import { EntityType } from '../../enums/entity.enum';
import { Bot, BotType } from '../../generated/entity/bot';
import { Bot, ProviderType } from '../../generated/entity/bot';
import { Operation } from '../../generated/entity/policies/accessControl/rule';
import { Include } from '../../generated/type/include';
import { Paging } from '../../generated/type/paging';
@ -140,13 +140,13 @@ const BotListV1 = ({
key: 'id',
width: 90,
render: (_, record) => {
const isIngestionBot = record.botType === BotType.IngestionBot;
const title = isIngestionBot
const isSystemBot = record.provider === ProviderType.System;
const title = isSystemBot
? INGESTION_BOT_CANT_BE_DELETED
: deletePermission
? 'Delete'
: NO_PERMISSION_FOR_ACTION;
const isDisabled = !deletePermission || isIngestionBot;
const isDisabled = !deletePermission || isSystemBot;
return (
<Space align="center" size={8}>