diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 1d7a94ccee1..35949d38ee3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -435,6 +435,21 @@ public interface CollectionDAO { default String getNameHashColumn() { return "fqnHash"; } + + @ConnectionAwareSqlQuery( + value = + "select JSON_EXTRACT(json, '$.fullyQualifiedName') from database_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseService' and toEntity = 'database')", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "select json ->> 'fullyQualifiedName' from database_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseService' and toEntity = 'database')", + connectionType = POSTGRES) + List getBrokenDatabase(); + + @SqlUpdate( + value = + "delete from database_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseService' and toEntity = 'database')") + int removeDatabase(); } interface DatabaseSchemaDAO extends EntityDAO { @@ -452,6 +467,21 @@ public interface CollectionDAO { default String getNameHashColumn() { return "fqnHash"; } + + @ConnectionAwareSqlQuery( + value = + "select JSON_EXTRACT(json, '$.fullyQualifiedName') from database_schema_entity where id not in (select toId from entity_relationship where fromEntity = 'database' and toEntity = 'databaseSchema')", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "select json ->> 'fullyQualifiedName' from database_schema_entity where id not in (select toId from entity_relationship where fromEntity = 'database' and toEntity = 'databaseSchema')", + connectionType = POSTGRES) + List getBrokenDatabaseSchemas(); + + @SqlUpdate( + value = + "delete from database_schema_entity where id not in (select toId from entity_relationship where fromEntity = 'database' and toEntity = 'databaseSchema')") + int removeBrokenDatabaseSchemas(); } interface DatabaseServiceDAO extends EntityDAO { @@ -3007,6 +3037,21 @@ public interface CollectionDAO { return "fqnHash"; } + @ConnectionAwareSqlQuery( + value = + "select JSON_EXTRACT(json, '$.fullyQualifiedName') from table_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseSchema' and toEntity = 'table')", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "select json ->> 'fullyQualifiedName' from table_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseSchema' and toEntity = 'table')", + connectionType = POSTGRES) + List getBrokenTables(); + + @SqlUpdate( + value = + "delete from table_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseSchema' and toEntity = 'table')") + int removeBrokenTables(); + @Override default int listCount(ListFilter filter) { String includeEmptyTestSuite = filter.getQueryParam("includeEmptyTestSuite"); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index e5b583281ac..64ed09f0504 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -646,6 +646,68 @@ public class OpenMetadataOperations implements Callable { } } + @Command( + name = "dbServiceCleanup", + description = "Cleans Up broken relationship hierarchy for database service.") + public Integer cleanupOrphanedEntities() { + try { + LOG.info("Running a Database Service Hierarchy Cleanup"); + parseConfig(); + + // Check Broken Tables + List brokenTables = Entity.getCollectionDAO().tableDAO().getBrokenTables(); + LOG.info("Following Tables seems to be Broken."); + List tableColumns = List.of(String.format("Tables(%d)", brokenTables.size())); + List> allRowsForTables = new ArrayList<>(); + for (String name : brokenTables) { + List row = new ArrayList<>(); + row.add(name); + allRowsForTables.add(row); + } + printToAsciiTable(tableColumns.stream().toList(), allRowsForTables, "No Broken Tables."); + LOG.info("Cleaning up the broken tables."); + if (!brokenTables.isEmpty()) { + Entity.getCollectionDAO().tableDAO().removeBrokenTables(); + } + + List brokenSchemas = + Entity.getCollectionDAO().databaseSchemaDAO().getBrokenDatabaseSchemas(); + LOG.info("Following DatabaseSchemas seems to be Broken."); + List dbSchemaColumns = + List.of(String.format("DatabaseSchemas(%d)", brokenSchemas.size())); + List> allRowsForSchemas = new ArrayList<>(); + for (String name : brokenSchemas) { + List row = new ArrayList<>(); + row.add(name); + allRowsForSchemas.add(row); + } + printToAsciiTable(dbSchemaColumns.stream().toList(), allRowsForSchemas, "No Broken Schemas."); + if (!brokenSchemas.isEmpty()) { + Entity.getCollectionDAO().databaseSchemaDAO().removeBrokenDatabaseSchemas(); + } + + List brokenDatabases = Entity.getCollectionDAO().databaseDAO().getBrokenDatabase(); + LOG.info("Following Database seems to be Broken."); + List databaseColumns = List.of(String.format("Database(%d)", brokenSchemas.size())); + List> allRowsForDatabases = new ArrayList<>(); + for (String name : brokenDatabases) { + List row = new ArrayList<>(); + row.add(name); + allRowsForDatabases.add(row); + } + printToAsciiTable( + databaseColumns.stream().toList(), allRowsForDatabases, "No Broken Databases."); + if (!brokenDatabases.isEmpty()) { + Entity.getCollectionDAO().databaseDAO().removeDatabase(); + } + + return 0; + } catch (Exception e) { + LOG.error("Failed to Entity Cleanup due to ", e); + return 1; + } + } + @Command(name = "reindex", description = "Re Indexes data into search engine from command line.") public Integer reIndex( @Option(