Fixes #8236 - Initialize bots from JSON data files (#8304)

bootstrap/sql/com.mysql.cj.jdbc.Driver/v006__create_db_connection_info.sql

@@ -9,4 +9,7 @@ CREATE TABLE IF NOT EXISTS web_analytic_event (
     deleted BOOLEAN GENERATED ALWAYS AS (json -> '$.deleted'),
     UNIQUE(name),
     INDEX name_index (name)
-);
+);
+
+UPDATE bot_entity
+SET json = JSON_INSERT(JSON_REMOVE(json, '$.botType'), '$.provider', 'system');

bootstrap/sql/org.postgresql.Driver/v006__create_db_connection_info.sql

@@ -11,3 +11,9 @@ CREATE TABLE IF NOT EXISTS web_analytic_event (
 );
 
 CREATE INDEX IF NOT EXISTS name_index ON web_analytic_event(name);
+
+UPDATE bot_entity
+SET json = JSONB_SET(json::jsonb, '{provider}', '"system"', true);
+
+UPDATE bot_entity
+SET json = json::jsonb #- '{botType}';

ingestion/src/metadata/ingestion/ometa/ometa_api.py

@@ -26,7 +26,6 @@ from pydantic import BaseModel
 from requests.utils import quote
 
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
-from metadata.generated.schema.entity.bot import BotType
 from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.entity.data.dashboard import Dashboard
 from metadata.generated.schema.entity.data.database import Database
@@ -180,7 +179,7 @@ class OpenMetadata(
 
         # Load auth provider config from Secret Manager if necessary
         self.secrets_manager_client.add_auth_provider_security_config(
-            self.config, BotType.ingestion_bot.value
+            self.config, "ingestion-bot"
         )
 
         # Load the auth provider init from the registry

ingestion/tests/unit/metadata/utils/secrets/test_aws_based_secrets_manager.py

@@ -17,7 +17,6 @@ from copy import deepcopy
 from typing import Any, Dict
 from unittest.mock import Mock, patch
 
-from metadata.generated.schema.entity.bot import BotType
 from metadata.generated.schema.entity.services.connections.serviceConnection import (
     ServiceConnection,
 )
@@ -88,7 +87,7 @@ class AWSBasedSecretsManager(object):
             actual_om_connection.securityConfig = None
 
             aws_manager.add_auth_provider_security_config(
-                actual_om_connection, BotType.ingestion_bot.value
+                actual_om_connection, "ingestion-bot"
             )
 
             self.assert_client_called_once(
@@ -111,7 +110,7 @@ class AWSBasedSecretsManager(object):
 
             with self.assertRaises(ValueError) as value_error:
                 aws_manager.add_auth_provider_security_config(
-                    self.om_connection, BotType.ingestion_bot.value
+                    self.om_connection, "ingestion-bot"
                 )
             self.assertTrue(
                 "/openmetadata/bot/ingestion-bot" in str(value_error.exception)

ingestion/tests/unit/metadata/utils/secrets/test_noop_secrets_manager.py

@@ -14,7 +14,6 @@ Test Local Secrets Manager
 """
 from copy import deepcopy
 
-from metadata.generated.schema.entity.bot import BotType
 from metadata.generated.schema.entity.services.connections.metadata.secretsManagerProvider import (
     SecretsManagerProvider,
 )
@@ -57,7 +56,7 @@ class TestLocalSecretsManager(TestSecretsManager.External):
         actual_om_connection.securityConfig = self.auth_provider_config
 
         noop_manager.add_auth_provider_security_config(
-            actual_om_connection, BotType.ingestion_bot.value
+            actual_om_connection, "ingestion-bot"
         )
 
         self.assertEqual(self.auth_provider_config, actual_om_connection.securityConfig)

openmetadata-service/src/main/java/org/openmetadata/service/Entity.java

@@ -130,7 +130,7 @@ public final class Entity {
   //
   public static final String ADMIN_USER_NAME = "admin";
   public static final String ORGANIZATION_NAME = "Organization";
-  public static final String DATACONSUMER_ROLE = "DataConsumer";
+  public static final String INGESTION_BOT_NAME = "ingestion-bot";
 
   //
   // List of entities whose changes should not be published to the Activity Feed

openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java

@@ -175,7 +175,7 @@ public final class CatalogExceptionMessage {
     return String.format("Failed to evaluate - %s", message);
   }
 
-  public static String deletionNotAllowed(String entityType, String name) {
+  public static String systemEntityDeleteNotAllowed(String name, String entityType) {
-    return String.format("Deletion of %s %s is not allowed", entityType, name);
+    return String.format("System entity [%s] of type %s can not be deleted.", name, entityType);
   }
 }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/BotRepository.java

@@ -15,12 +15,13 @@ package org.openmetadata.service.jdbi3;
 
 import java.io.IOException;
 import org.openmetadata.schema.entity.Bot;
-import org.openmetadata.schema.entity.BotType;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
+import org.openmetadata.schema.type.ProviderType;
 import org.openmetadata.schema.type.Relationship;
 import org.openmetadata.service.Entity;
+import org.openmetadata.service.exception.CatalogExceptionMessage;
 import org.openmetadata.service.resources.bots.BotResource;
 import org.openmetadata.service.secrets.SecretsManager;
 import org.openmetadata.service.secrets.SecretsManagerFactory;
@@ -51,9 +52,9 @@ public class BotRepository extends EntityRepository<Bot> {
     EntityReference botUser = entity.getBotUser();
     entity.withBotUser(null);
     store(entity.getId(), entity, update);
-    if (!BotType.BOT.equals(entity.getBotType())) {
+    if (!ProviderType.USER.equals(entity.getProvider())) { // encryption is done only for system bots
       SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
-      secretsManager.encryptOrDecryptBotCredentials(entity.getBotType().value(), botUser.getName(), true);
+      secretsManager.encryptOrDecryptBotCredentials(entity.getName(), botUser.getName(), true);
     }
     entity.withBotUser(botUser);
   }
@@ -68,6 +69,14 @@ public class BotRepository extends EntityRepository<Bot> {
     return new BotUpdater(original, updated, operation);
   }
 
+  @Override
+  protected void preDelete(Bot entity) {
+    if (entity.getProvider() == ProviderType.SYSTEM) { // System provided bot can't be deleted
+      throw new IllegalArgumentException(
+          CatalogExceptionMessage.systemEntityDeleteNotAllowed(entity.getName(), Entity.BOT));
+    }
+  }
+
   @Override
   public void restorePatchAttributes(Bot original, Bot updated) {
     // Bot user can't be changed by patch
@@ -86,9 +95,6 @@ public class BotRepository extends EntityRepository<Bot> {
     @Override
     public void entitySpecificUpdate() throws IOException {
       updateUser(original, updated);
-      if (original.getBotType() != null) {
-        updated.setBotType(original.getBotType());
-      }
     }
 
     private void updateUser(Bot original, Bot updated) throws IOException {

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java

@@ -254,16 +254,16 @@ public abstract class EntityRepository<T extends EntityInterface> {
   }
 
   public List<T> getEntitiesFromSeedData(String path) throws IOException {
-    return getEntitiesFromSeedData(path, entityClass);
+    return getEntitiesFromSeedData(entityType, path, entityClass);
   }
 
-  public <U> List<U> getEntitiesFromSeedData(String path, Class<U> clazz) throws IOException {
+  public static <U> List<U> getEntitiesFromSeedData(String entityType, String path, Class<U> clazz) throws IOException {
     List<U> entities = new ArrayList<>();
     List<String> jsonDataFiles = EntityUtil.getJsonDataResources(path);
     jsonDataFiles.forEach(
         jsonDataFile -> {
           try {
-            String json = CommonUtil.getResourceAsStream(getClass().getClassLoader(), jsonDataFile);
+            String json = CommonUtil.getResourceAsStream(EntityRepository.class.getClassLoader(), jsonDataFile);
             json = json.replace("<separator>", Entity.SEPARATOR);
             entities.add(JsonUtils.readValue(json, clazz));
           } catch (Exception e) {

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PolicyRepository.java

@@ -143,7 +143,8 @@ public class PolicyRepository extends EntityRepository<Policy> {
   @Override
   protected void preDelete(Policy entity) {
     if (FALSE.equals(entity.getAllowDelete())) {
-      throw new IllegalArgumentException(CatalogExceptionMessage.deletionNotAllowed(Entity.POLICY, entity.getName()));
+      throw new IllegalArgumentException(
+          CatalogExceptionMessage.systemEntityDeleteNotAllowed(entity.getName(), Entity.POLICY));
     }
   }
 

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/RoleRepository.java

@@ -114,7 +114,8 @@ public class RoleRepository extends EntityRepository<Role> {
   @Override
   protected void preDelete(Role entity) {
     if (FALSE.equals(entity.getAllowDelete())) {
-      throw new IllegalArgumentException(CatalogExceptionMessage.deletionNotAllowed(Entity.ROLE, entity.getName()));
+      throw new IllegalArgumentException(
+          CatalogExceptionMessage.systemEntityDeleteNotAllowed(entity.getName(), Entity.ROLE));
     }
   }
 

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java

@@ -93,7 +93,7 @@ public class UserRepository extends EntityRepository<User> {
 
   private List<EntityReference> getInheritedRoles(User user) throws IOException {
     getTeams(user);
-    return SubjectCache.getInstance().getRolesForTeams(getTeams(user));
+    return SubjectCache.getInstance() != null ? SubjectCache.getInstance().getRolesForTeams(getTeams(user)) : null;
   }
 
   @Override

openmetadata-service/src/main/java/org/openmetadata/service/resources/Collection.java

@@ -23,6 +23,6 @@ import java.lang.annotation.Target;
 public @interface Collection {
   String name();
 
-  /** Only order from 0 to 9 (inclusive) are allowed */
+  /** Order of initialization of resource starting from 0. Only order from 0 to 9 (inclusive) are allowed */
   int order() default 9;
 }

openmetadata-service/src/main/java/org/openmetadata/service/resources/CollectionRegistry.java

@@ -264,8 +264,10 @@ public final class CollectionRegistry {
     try {
       Method initializeMethod = resource.getClass().getMethod("initialize", OpenMetadataApplicationConfig.class);
       initializeMethod.invoke(resource, config);
-    } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException ignored) {
+    } catch (NoSuchMethodException ignored) {
       // Method does not exist and initialize is not called
+    } catch (Exception ex) {
+      LOG.warn("Encountered exception ", ex);
     }
     return resource;
   }

openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotResource.java

@@ -13,6 +13,8 @@
 
 package org.openmetadata.service.resources.bots;
 
+import static org.openmetadata.service.security.DefaultAuthorizer.user;
+
 import io.swagger.annotations.Api;
 import io.swagger.v3.oas.annotations.ExternalDocumentation;
 import io.swagger.v3.oas.annotations.Operation;
@@ -45,14 +47,16 @@ import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.SecurityContext;
 import javax.ws.rs.core.UriInfo;
+import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.schema.api.CreateBot;
 import org.openmetadata.schema.entity.Bot;
-import org.openmetadata.schema.entity.BotType;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.type.EntityHistory;
 import org.openmetadata.schema.type.Include;
+import org.openmetadata.schema.type.ProviderType;
 import org.openmetadata.schema.type.Relationship;
 import org.openmetadata.service.Entity;
+import org.openmetadata.service.OpenMetadataApplicationConfig;
 import org.openmetadata.service.jdbi3.BotRepository;
 import org.openmetadata.service.jdbi3.CollectionDAO;
 import org.openmetadata.service.jdbi3.ListFilter;
@@ -62,9 +66,12 @@ import org.openmetadata.service.resources.EntityResource;
 import org.openmetadata.service.secrets.SecretsManager;
 import org.openmetadata.service.secrets.SecretsManagerFactory;
 import org.openmetadata.service.security.Authorizer;
+import org.openmetadata.service.security.DefaultAuthorizer;
+import org.openmetadata.service.security.SecurityUtil;
 import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.ResultList;
 
+@Slf4j
 @Path("/v1/bots")
 @Api(value = "Bot collection", tags = "Bot collection")
 @Produces(MediaType.APPLICATION_JSON)
@@ -77,6 +84,23 @@ public class BotResource extends EntityResource<Bot, BotRepository> {
     super(Bot.class, new BotRepository(dao), authorizer);
   }
 
+  @SuppressWarnings("unused") // Method is used by reflection
+  public void initialize(OpenMetadataApplicationConfig config) throws IOException {
+    // Load system bots
+    List<Bot> bots = dao.getEntitiesFromSeedData();
+    String domain = SecurityUtil.getDomain(config);
+    for (Bot bot : bots) {
+      String userName = bot.getBotUser().getName();
+      User user = user(userName, domain, userName).withIsBot(true).withIsAdmin(false);
+      user = DefaultAuthorizer.addOrUpdateBotUser(user, config);
+      bot.withId(UUID.randomUUID())
+          .withBotUser(user.getEntityReference())
+          .withUpdatedBy(userName)
+          .withUpdatedAt(System.currentTimeMillis());
+      dao.initializeEntity(bot);
+    }
+  }
+
   @Override
   public Bot addHref(UriInfo uriInfo, Bot entity) {
     Entity.withHref(uriInfo, entity.getBotUser());
@@ -265,10 +289,11 @@ public class BotResource extends EntityResource<Bot, BotRepository> {
     Bot bot = getBot(securityContext, create);
     Response response = createOrUpdate(uriInfo, securityContext, bot, false);
     // ensures the secrets' manager store the credentials even when the botUser does not change
+    // TODO encrypt decrypt could be done twice
     bot = (Bot) response.getEntity();
     SecretsManager secretsManager = SecretsManagerFactory.getSecretsManager();
-    if (!BotType.BOT.equals(bot.getBotType())) {
+    if (!ProviderType.USER.equals(bot.getProvider())) { // User bots credentials are not stored. Only system bots.
-      secretsManager.encryptOrDecryptBotCredentials(bot.getBotType().value(), bot.getBotUser().getName(), true);
+      secretsManager.encryptOrDecryptBotCredentials(bot.getName(), bot.getBotUser().getName(), true);
     }
     return response;
   }
@@ -319,17 +344,13 @@ public class BotResource extends EntityResource<Bot, BotRepository> {
           boolean hardDelete,
       @Parameter(description = "Id of the Bot", schema = @Schema(type = "UUID")) @PathParam("id") UUID id)
       throws IOException {
-    BotType botType = dao.get(null, id, EntityUtil.Fields.EMPTY_FIELDS).getBotType();
-    if (!BotType.BOT.equals(botType)) {
-      throw new IllegalArgumentException(String.format("[%s] can not be deleted.", botType.value()));
-    }
     return delete(uriInfo, securityContext, id, true, hardDelete, false);
   }
 
   private Bot getBot(CreateBot create, String user) throws IOException {
     return copy(new Bot(), create, user)
         .withBotUser(create.getBotUser())
-        .withBotType(BotType.BOT)
+        .withProvider(create.getProvider())
         .withFullyQualifiedName(create.getName());
   }
 

openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java

@@ -95,7 +95,7 @@ public class PolicyResource extends EntityResource<Policy, PolicyRepository> {
     super(Policy.class, new PolicyRepository(dao), authorizer);
   }
 
-  @SuppressWarnings("unused") // Method is used for reflection
+  @SuppressWarnings("unused") // Method is used by reflection
   public void initialize(OpenMetadataApplicationConfig config) throws IOException {
     // Load any existing rules from database, before loading seed data.
     dao.initSeedDataFromResources();

openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java

@@ -93,7 +93,8 @@ public class TagResource {
   @SuppressWarnings("unused") // Method used by reflection
   public void initialize(OpenMetadataApplicationConfig config) throws IOException {
     // Find tag definitions and load tag categories from the json file, if necessary
-    List<TagCategory> tagCategories = dao.getEntitiesFromSeedData(".*json/data/tags/.*\\.json$", TagCategory.class);
+    List<TagCategory> tagCategories =
+        dao.getEntitiesFromSeedData(Entity.TAG_CATEGORY, ".*json/data/tags/.*\\.json$", TagCategory.class);
     for (TagCategory tagCategory : tagCategories) {
       long now = System.currentTimeMillis();
       tagCategory.withId(UUID.randomUUID()).withUpdatedBy(ADMIN_USER_NAME).withUpdatedAt(now);

openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java

@@ -141,7 +141,7 @@ import org.openmetadata.service.util.TokenUtil;
 @Api(value = "User collection", tags = "User collection")
 @Produces(MediaType.APPLICATION_JSON)
 @Consumes(MediaType.APPLICATION_JSON)
-@Collection(name = "users")
+@Collection(name = "users", order = 8) // Initialize user resource before bot resource (at default order 9)
 public class UserResource extends EntityResource<User, UserRepository> {
   public static final String COLLECTION_PATH = "v1/users/";
   public static final String USER_PROTECTED_FIELDS = "authenticationMechanism";

openmetadata-service/src/main/java/org/openmetadata/service/security/DefaultAuthorizer.java

@@ -23,27 +23,21 @@ import static org.openmetadata.schema.teams.authn.SSOAuthMechanism.SsoServiceTyp
 import static org.openmetadata.service.Entity.ADMIN_USER_NAME;
 import static org.openmetadata.service.exception.CatalogExceptionMessage.notAdmin;
 import static org.openmetadata.service.resources.teams.UserResource.USER_PROTECTED_FIELDS;
-import static org.openmetadata.service.security.SecurityUtil.DEFAULT_PRINCIPAL_DOMAIN;
 
 import at.favre.lib.crypto.bcrypt.BCrypt;
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
-import java.util.stream.Collectors;
 import javax.ws.rs.core.SecurityContext;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.jdbi.v3.core.Jdbi;
 import org.openmetadata.schema.api.configuration.airflow.AirflowConfiguration;
 import org.openmetadata.schema.api.security.AuthenticationConfiguration;
 import org.openmetadata.schema.email.SmtpSettings;
-import org.openmetadata.schema.entity.Bot;
-import org.openmetadata.schema.entity.BotType;
 import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.security.client.OpenMetadataJWTClientConfig;
@@ -57,7 +51,6 @@ import org.openmetadata.schema.type.ResourcePermission;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.OpenMetadataApplicationConfig;
 import org.openmetadata.service.exception.EntityNotFoundException;
-import org.openmetadata.service.jdbi3.BotRepository;
 import org.openmetadata.service.jdbi3.EntityRepository;
 import org.openmetadata.service.jdbi3.UserRepository;
 import org.openmetadata.service.secrets.SecretsManager;
@@ -78,88 +71,52 @@ import org.openmetadata.service.util.RestUtil;
 @Slf4j
 public class DefaultAuthorizer implements Authorizer {
   private static final String COLON_DELIMITER = ":";
-  private static final String DEFAULT_ADMIN = ADMIN_USER_NAME;
-  private Set<String> adminUsers;
-  private Set<String> botPrincipalUsers;
-  private Set<String> testUsers;
-  private String principalDomain;
-  private String providerType;
   private boolean isSmtpEnabled;
 
   @Override
-  public void init(OpenMetadataApplicationConfig openMetadataApplicationConfig, Jdbi dbi) {
+  public void init(OpenMetadataApplicationConfig config, Jdbi dbi) {
-    LOG.info(
+    LOG.info("Initializing DefaultAuthorizer with config {}", config.getAuthorizerConfiguration());
-        "Initializing DefaultAuthorizer with config {}", openMetadataApplicationConfig.getAuthorizerConfiguration());
+    SmtpSettings smtpSettings = config.getSmtpSettings();
-    this.adminUsers = new HashSet<>(openMetadataApplicationConfig.getAuthorizerConfiguration().getAdminPrincipals());
-    this.botPrincipalUsers =
-        new HashSet<>(openMetadataApplicationConfig.getAuthorizerConfiguration().getBotPrincipals());
-    this.testUsers = new HashSet<>(openMetadataApplicationConfig.getAuthorizerConfiguration().getTestPrincipals());
-    this.principalDomain = openMetadataApplicationConfig.getAuthorizerConfiguration().getPrincipalDomain();
-    this.providerType = openMetadataApplicationConfig.getAuthenticationConfiguration().getProvider();
-    SmtpSettings smtpSettings = openMetadataApplicationConfig.getSmtpSettings();
     this.isSmtpEnabled = smtpSettings != null && smtpSettings.getEnableSmtpServer();
     SubjectCache.initialize();
     PolicyCache.initialize();
     RoleCache.initialize();
-    LOG.debug("Admin users: {}", adminUsers);
+    initializeUsers(config);
-    initializeUsers(openMetadataApplicationConfig);
   }
 
-  private void initializeUsers(OpenMetadataApplicationConfig openMetadataApplicationConfig) {
+  private void initializeUsers(OpenMetadataApplicationConfig config) {
-    LOG.debug("Checking user entries for admin users");
+    Set<String> adminUsers = new HashSet<>(config.getAuthorizerConfiguration().getAdminPrincipals());
-    String domain = principalDomain.isEmpty() ? DEFAULT_PRINCIPAL_DOMAIN : principalDomain;
+    LOG.debug("Checking user entries for admin users {}", adminUsers);
-    if (!providerType.equals(SSOAuthMechanism.SsoServiceType.BASIC.value())) {
+    String domain = SecurityUtil.getDomain(config);
-      for (String adminUser : adminUsers) {
+    String providerType = config.getAuthenticationConfiguration().getProvider();
-        User user = user(adminUser, domain, adminUser).withIsAdmin(true);
+    if (providerType.equals(SSOAuthMechanism.SsoServiceType.BASIC.value())) {
-        addOrUpdateUser(user);
+      handleBasicAuth(adminUsers, domain);
-      }
     } else {
-      try {
+      addUsers(adminUsers, domain, true);
-        handleBasicAuth(adminUsers, domain);
-      } catch (IOException e) {
-        LOG.error("Failed in Basic Auth Setup. Reason : {}", e.getMessage());
-      }
-    }
-
-    LOG.debug("Checking user entries for bot users");
-    Set<String> botUsers = Arrays.stream(BotType.values()).map(BotType::value).collect(Collectors.toSet());
-    botUsers.remove(BotType.BOT.value());
-    botUsers.addAll(botPrincipalUsers);
-    for (String botUser : botUsers) {
-      User user = user(botUser, domain, botUser).withIsBot(true).withIsAdmin(false);
-      user = addOrUpdateBotUser(user, openMetadataApplicationConfig);
-      if (user != null) {
-        BotType botType;
-        try {
-          botType = BotType.fromValue(botUser);
-        } catch (IllegalArgumentException e) {
-          botType = BotType.BOT;
-        }
-        Bot bot = bot(user).withBotUser(user.getEntityReference()).withBotType(botType);
-        addOrUpdateBot(bot);
-      }
     }
 
     LOG.debug("Checking user entries for test users");
-    for (String testUser : testUsers) {
+    Set<String> testUsers = new HashSet<>(config.getAuthorizerConfiguration().getTestPrincipals());
-      User user = user(testUser, domain, testUser);
+    addUsers(testUsers, domain, null);
-      addOrUpdateUser(user);
-    }
   }
 
-  private void handleBasicAuth(Set<String> adminUsers, String domain) throws IOException {
+  private void handleBasicAuth(Set<String> adminUsers, String domain) {
-    for (String adminUser : adminUsers) {
+    try {
-      if (adminUser.contains(COLON_DELIMITER)) {
+      for (String adminUser : adminUsers) {
-        String[] tokens = adminUser.split(COLON_DELIMITER);
+        if (adminUser.contains(COLON_DELIMITER)) {
-        addUserForBasicAuth(tokens[0], tokens[1], domain);
+          String[] tokens = adminUser.split(COLON_DELIMITER);
-      } else {
+          addUserForBasicAuth(tokens[0], tokens[1], domain);
-        boolean isDefaultAdmin = adminUser.equals(DEFAULT_ADMIN);
+        } else {
-        String token = PasswordUtil.generateRandomPassword();
+          boolean isDefaultAdmin = adminUser.equals(ADMIN_USER_NAME);
-        if (isDefaultAdmin || !isSmtpEnabled) {
+          String token = PasswordUtil.generateRandomPassword();
-          token = DEFAULT_ADMIN;
+          if (isDefaultAdmin || !isSmtpEnabled) {
+            token = ADMIN_USER_NAME;
+          }
+          addUserForBasicAuth(adminUser, token, domain);
         }
-        addUserForBasicAuth(adminUser, token, domain);
       }
+    } catch (IOException e) {
+      LOG.error("Failed in Basic Auth Setup. Reason : {}", e.getMessage());
     }
   }
 
@@ -213,7 +170,7 @@ public class DefaultAuthorizer implements Authorizer {
             .withConfig(new BasicAuthMechanism().withPassword(hashedPwd)));
   }
 
-  private User user(String name, String domain, String updatedBy) {
+  public static User user(String name, String domain, String updatedBy) {
     return new User()
         .withId(UUID.randomUUID())
         .withName(name)
@@ -224,16 +181,6 @@ public class DefaultAuthorizer implements Authorizer {
         .withIsBot(false);
   }
 
-  private Bot bot(User user) {
-    return new Bot()
-        .withId(UUID.randomUUID())
-        .withName(user.getName())
-        .withFullyQualifiedName(user.getName())
-        .withUpdatedBy(user.getUpdatedBy())
-        .withUpdatedAt(System.currentTimeMillis())
-        .withDisplayName(user.getName());
-  }
-
   @Override
   public List<ResourcePermission> listPermissions(SecurityContext securityContext, String user) {
     SubjectContext subjectContext = getSubjectContext(securityContext);
@@ -307,7 +254,14 @@ public class DefaultAuthorizer implements Authorizer {
     throw new AuthorizationException(notAdmin(securityContext.getUserPrincipal().getName()));
   }
 
-  private User addOrUpdateUser(User user) {
+  private void addUsers(Set<String> users, String domain, Boolean isAdmin) {
+    for (String userName : users) {
+      User user = user(userName, domain, userName).withIsAdmin(isAdmin);
+      addOrUpdateUser(user);
+    }
+  }
+
+  private static User addOrUpdateUser(User user) {
     EntityRepository<User> userRepository = Entity.getEntityRepository(Entity.USER);
     try {
       RestUtil.PutResponse<User> addedUser = userRepository.createOrUpdate(null, user);
@@ -316,7 +270,7 @@ public class DefaultAuthorizer implements Authorizer {
       return addedUser.getEntity();
     } catch (Exception exception) {
       // In HA set up the other server may have already added the user.
-      LOG.debug("Caught exception: {}", ExceptionUtils.getStackTrace(exception));
+      LOG.debug("Caught exception", exception);
       user.setAuthenticationMechanism(null);
       LOG.debug("User entry: {} already exists.", user.getName());
     }
@@ -341,7 +295,7 @@ public class DefaultAuthorizer implements Authorizer {
    *       </ul>
    * </ul>
    */
-  private User addOrUpdateBotUser(User user, OpenMetadataApplicationConfig openMetadataApplicationConfig) {
+  public static User addOrUpdateBotUser(User user, OpenMetadataApplicationConfig openMetadataApplicationConfig) {
     User originalUser = retrieveAuthMechanism(user);
     // the user did not have an auth mechanism
     AuthenticationMechanism authMechanism = originalUser != null ? originalUser.getAuthenticationMechanism() : null;
@@ -402,15 +356,16 @@ public class DefaultAuthorizer implements Authorizer {
     return addOrUpdateUser(user);
   }
 
-  private SSOAuthMechanism buildAuthMechanismConfig(SSOAuthMechanism.SsoServiceType ssoServiceType, Object config) {
+  private static SSOAuthMechanism buildAuthMechanismConfig(
+      SSOAuthMechanism.SsoServiceType ssoServiceType, Object config) {
     return new SSOAuthMechanism().withSsoServiceType(ssoServiceType).withAuthConfig(config);
   }
 
-  private AuthenticationMechanism buildAuthMechanism(AuthenticationMechanism.AuthType authType, Object config) {
+  private static AuthenticationMechanism buildAuthMechanism(AuthenticationMechanism.AuthType authType, Object config) {
     return new AuthenticationMechanism().withAuthType(authType).withConfig(config);
   }
 
-  private User retrieveAuthMechanism(User user) {
+  private static User retrieveAuthMechanism(User user) {
     EntityRepository<User> userRepository = UserRepository.class.cast(Entity.getEntityRepository(Entity.USER));
     try {
       User originalUser =
@@ -429,24 +384,6 @@ public class DefaultAuthorizer implements Authorizer {
     }
   }
 
-  private void addOrUpdateBot(Bot bot) {
-    EntityRepository<Bot> botRepository = BotRepository.class.cast(Entity.getEntityRepository(Entity.BOT));
-    Bot originalBot;
-    try {
-      originalBot = botRepository.getByName(null, bot.getName(), EntityUtil.Fields.EMPTY_FIELDS);
-      bot.setBotUser(originalBot.getBotUser());
-    } catch (Exception e) {
-    }
-    try {
-      RestUtil.PutResponse<Bot> addedBot = botRepository.createOrUpdate(null, bot);
-      LOG.debug("Added bot entry: {}", addedBot.getEntity().getName());
-    } catch (Exception exception) {
-      // In HA set up the other server may have already added the bot.
-      LOG.debug("Caught exception: {}", ExceptionUtils.getStackTrace(exception));
-      LOG.debug("Bot entry: {} already exists.", bot.getName());
-    }
-  }
-
   private SubjectContext getSubjectContext(SecurityContext securityContext) {
     if (securityContext == null || securityContext.getUserPrincipal() == null) {
       throw new AuthenticationException("No principal in security context");

openmetadata-service/src/main/java/org/openmetadata/service/security/NoopAuthorizer.java

@@ -18,7 +18,6 @@ import java.util.List;
 import java.util.UUID;
 import javax.ws.rs.core.SecurityContext;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.jdbi.v3.core.Jdbi;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.type.EntityReference;
@@ -99,7 +98,7 @@ public class NoopAuthorizer implements Authorizer {
       LOG.debug("Added anonymous user entry: {}", addedUser);
     } catch (IOException exception) {
       // In HA set up the other server may have already added the user.
-      LOG.debug("Caught exception: {}", ExceptionUtils.getStackTrace(exception));
+      LOG.debug("Caught exception ", exception);
       LOG.debug("Anonymous user entry: {} already exists.", user);
     }
   }

openmetadata-service/src/main/java/org/openmetadata/service/security/SecurityUtil.java

@@ -19,6 +19,8 @@ import java.security.Principal;
 import java.util.Map;
 import javax.ws.rs.client.Invocation;
 import javax.ws.rs.client.WebTarget;
+import org.openmetadata.common.utils.CommonUtil;
+import org.openmetadata.service.OpenMetadataApplicationConfig;
 
 public final class SecurityUtil {
   public static final String DEFAULT_PRINCIPAL_DOMAIN = "openmetadata.org";
@@ -46,6 +48,11 @@ public final class SecurityUtil {
     return principal == null ? null : principal.split("@")[0];
   }
 
+  public static String getDomain(OpenMetadataApplicationConfig config) {
+    String principalDomain = config.getAuthorizerConfiguration().getPrincipalDomain();
+    return CommonUtil.nullOrEmpty(principalDomain) ? DEFAULT_PRINCIPAL_DOMAIN : principalDomain;
+  }
+
   public static Invocation.Builder addHeaders(WebTarget target, Map<String, String> headers) {
     if (headers != null) {
       return target

openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataServerConnectionBuilder.java

@@ -6,7 +6,6 @@ import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.api.configuration.airflow.SSLConfig;
 import org.openmetadata.schema.api.configuration.airflow.AirflowConfiguration;
 import org.openmetadata.schema.entity.Bot;
-import org.openmetadata.schema.entity.BotType;
 import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.security.client.OpenMetadataJWTClientConfig;
@@ -109,7 +108,7 @@ public class OpenMetadataServerConnectionBuilder {
   }
 
   private User retrieveBotUser() {
-    User botUser = retrieveIngestionBotUser(BotType.INGESTION_BOT.value());
+    User botUser = retrieveIngestionBotUser(Entity.INGESTION_BOT_NAME);
     if (botUser == null) {
       throw new IllegalArgumentException("Please, verify that the ingestion-bot is present.");
     }

openmetadata-service/src/main/java/org/openmetadata/service/util/TablesInitializer.java

@@ -50,6 +50,7 @@ import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.teams.authn.GenerateTokenRequest;
 import org.openmetadata.schema.teams.authn.JWTAuthMechanism;
 import org.openmetadata.schema.teams.authn.JWTTokenExpiry;
+import org.openmetadata.service.Entity;
 import org.openmetadata.service.OpenMetadataApplicationConfig;
 import org.openmetadata.service.elasticsearch.ElasticSearchIndexDefinition;
 import org.openmetadata.service.fernet.Fernet;
@@ -57,6 +58,7 @@ import org.openmetadata.service.jdbi3.CollectionDAO;
 import org.openmetadata.service.jdbi3.UserRepository;
 import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator;
 import org.openmetadata.service.secrets.SecretsManagerFactory;
+import org.openmetadata.service.security.SecurityUtil;
 import org.openmetadata.service.security.jwt.JWTTokenGenerator;
 
 public final class TablesInitializer {
@@ -333,11 +335,8 @@ public final class TablesInitializer {
   }
 
   private static void createIngestionBot(OpenMetadataApplicationConfig config, Jdbi jdbi) {
-    String domain =
+    String domain = SecurityUtil.getDomain(config);
-        config.getAuthorizerConfiguration().getPrincipalDomain().isEmpty()
+    String botUser = Entity.INGESTION_BOT_NAME;
-            ? DEFAULT_PRINCIPAL_DOMAIN
-            : config.getAuthorizerConfiguration().getPrincipalDomain();
-    String botUser = "ingestion-bot";
 
     User user =
         new User()

openmetadata-service/src/main/resources/json/data/bot/ingestionBot.json

@@ -0,0 +1,10 @@
+{
+  "name": "ingestion-bot",
+  "description": "Bot used for ingesting metadata.",
+  "fullyQualifiedName": "ingestion-bot",
+  "botUser": {
+    "name" : "ingestion-bot",
+    "type" : "user"
+  },
+  "provider": "system"
+}

openmetadata-service/src/test/java/org/openmetadata/service/resources/bots/BotResourceTest.java

@@ -1,11 +1,13 @@
 package org.openmetadata.service.resources.bots;
 
 import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
 import static org.openmetadata.service.util.TestUtils.assertResponse;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import lombok.SneakyThrows;
@@ -16,10 +18,11 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
 import org.openmetadata.schema.api.CreateBot;
 import org.openmetadata.schema.entity.Bot;
-import org.openmetadata.schema.entity.BotType;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.service.Entity;
+import org.openmetadata.service.exception.CatalogExceptionMessage;
+import org.openmetadata.service.jdbi3.EntityRepository;
 import org.openmetadata.service.resources.EntityResourceTest;
 import org.openmetadata.service.resources.bots.BotResource.BotList;
 import org.openmetadata.service.resources.teams.UserResourceTest;
@@ -52,6 +55,15 @@ public class BotResourceTest extends EntityResourceTest<Bot, CreateBot> {
     }
   }
 
+  @Test
+  void testBotInitialization() throws IOException {
+    // Ensure all the bots are bootstrapped from the data files
+    List<Bot> bots = EntityRepository.getEntitiesFromSeedData(Entity.BOT, ".*json/data/bot/.*\\.json$", Bot.class);
+    for (Bot bot : bots) {
+      assertNotNull(getEntityByName(bot.getName(), "", ADMIN_AUTH_HEADERS));
+    }
+  }
+
   @Test
   void put_entityNonEmptyDescriptionUpdate_200(TestInfo test) {
     // PUT based updates are categorized as create operation
@@ -103,14 +115,14 @@ public class BotResourceTest extends EntityResourceTest<Bot, CreateBot> {
   }
 
   @Test
-  void delete_failIfUserIsIngestionBot(TestInfo test) throws IOException {
+  void delete_failIfUserIsIngestionBot() throws IOException {
     // get ingestion bot
-    Bot ingestionBot = getEntityByName(BotType.INGESTION_BOT.value(), "", ADMIN_AUTH_HEADERS);
+    Bot ingestionBot = getEntityByName(Entity.INGESTION_BOT_NAME, "", ADMIN_AUTH_HEADERS);
     // fail because it is a system bot
     assertResponse(
         () -> deleteEntity(ingestionBot.getId(), true, true, ADMIN_AUTH_HEADERS),
         BAD_REQUEST,
-        "[ingestion-bot] can not be deleted.");
+        CatalogExceptionMessage.systemEntityDeleteNotAllowed(ingestionBot.getName(), Entity.BOT));
   }
 
   @Override

openmetadata-service/src/test/java/org/openmetadata/service/resources/policies/PolicyResourceTest.java

@@ -243,7 +243,7 @@ public class PolicyResourceTest extends EntityResourceTest<Policy, CreatePolicy>
       assertResponse(
           () -> deleteEntity(policy.getId(), ADMIN_AUTH_HEADERS),
           BAD_REQUEST,
-          CatalogExceptionMessage.deletionNotAllowed(Entity.POLICY, policy.getName()));
+          CatalogExceptionMessage.systemEntityDeleteNotAllowed(policy.getName(), Entity.POLICY));
     }
   }
 

openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/RoleResourceTest.java

@@ -142,7 +142,7 @@ public class RoleResourceTest extends EntityResourceTest<Role, CreateRole> {
       assertResponse(
           () -> deleteEntity(role.getId(), ADMIN_AUTH_HEADERS),
           BAD_REQUEST,
-          CatalogExceptionMessage.deletionNotAllowed(Entity.ROLE, role.getName()));
+          CatalogExceptionMessage.systemEntityDeleteNotAllowed(role.getName(), Entity.ROLE));
     }
   }
 

openmetadata-spec/src/main/resources/json/schema/api/createBot.json

@@ -23,6 +23,10 @@
     "description": {
       "description": "Description of the bot.",
       "type": "string"
+    },
+    "provider" : {
+      "$ref": "../type/basic.json#/definitions/providerType",
+      "default": "user"
     }
   },
   "required": ["name", "botUser"],

openmetadata-spec/src/main/resources/json/schema/entity/bot.json

@@ -6,15 +6,6 @@
   "type": "object",
   "javaType": "org.openmetadata.schema.entity.Bot",
   "javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
-  "definitions": {
-    "botType": {
-      "javaType": "org.openmetadata.schema.entity.BotType",
-      "description": "Type of bot",
-      "type": "string",
-      "enum": ["bot", "ingestion-bot"],
-      "default": "bot"
-    }
-  },
   "properties": {
     "id": {
       "description": "Unique identifier of a bot instance.",
@@ -40,9 +31,9 @@
       "description": "Bot user created for this bot on behalf of which the bot performs all the operations, such as updating description, responding on the conversation threads, etc.",
       "$ref" : "../type/entityReference.json"
     },
-    "botType" : {
+    "provider" : {
-      "$ref": "#/definitions/botType",
+      "$ref": "../type/basic.json#/definitions/providerType",
-      "default": "bot"
+      "default": "user"
     },
     "version": {
       "description": "Metadata version of the entity.",

openmetadata-spec/src/main/resources/json/schema/type/basic.json

@@ -120,6 +120,13 @@
     },
     "entityExtension" : {
       "description": "Entity extension data with custom attributes added to the entity."
+    },
+    "providerType": {
+      "javaType": "org.openmetadata.schema.type.ProviderType",
+      "description": "Type of provider of an entity. Some entities are provided by the `system`. Some are are entities created and provided by the `user`. Typically `system` provide entities can't be deleted and can only be disabled.",
+      "type": "string",
+      "enum": ["system", "user"],
+      "default": "user"
     }
   }
 }

openmetadata-ui/src/main/resources/ui/src/components/BotListV1/BotListV1.component.tsx

@@ -29,7 +29,7 @@ import {
   NO_PERMISSION_FOR_ACTION,
 } from '../../constants/HelperTextUtil';
 import { EntityType } from '../../enums/entity.enum';
-import { Bot, BotType } from '../../generated/entity/bot';
+import { Bot, ProviderType } from '../../generated/entity/bot';
 import { Operation } from '../../generated/entity/policies/accessControl/rule';
 import { Include } from '../../generated/type/include';
 import { Paging } from '../../generated/type/paging';
@@ -140,13 +140,13 @@ const BotListV1 = ({
         key: 'id',
         width: 90,
         render: (_, record) => {
-          const isIngestionBot = record.botType === BotType.IngestionBot;
+          const isSystemBot = record.provider === ProviderType.System;
-          const title = isIngestionBot
+          const title = isSystemBot
             ? INGESTION_BOT_CANT_BE_DELETED
             : deletePermission
             ? 'Delete'
             : NO_PERMISSION_FOR_ACTION;
-          const isDisabled = !deletePermission || isIngestionBot;
+          const isDisabled = !deletePermission || isSystemBot;
 
           return (
             <Space align="center" size={8}>