yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-12 17:26:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup operations for database services (#21006)

Browse Source
This commit is contained in:
Mohit Yadav 2025-04-29 12:04:02 +05:30 committed by GitHub
parent d901dd2948
commit 575c91fcc9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 107 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View File

@ -435,6 +435,21 @@ public interface CollectionDAO {
default String getNameHashColumn() {
return "fqnHash";
}
@ConnectionAwareSqlQuery(
value =
"select JSON_EXTRACT(json, '$.fullyQualifiedName') from database_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseService' and toEntity = 'database')",
connectionType = MYSQL)
@ConnectionAwareSqlQuery(
value =
"select json ->> 'fullyQualifiedName' from database_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseService' and toEntity = 'database')",
connectionType = POSTGRES)
List<String> getBrokenDatabase();
@SqlUpdate(
value =
"delete from database_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseService' and toEntity = 'database')")
int removeDatabase();
}
interface DatabaseSchemaDAO extends EntityDAO<DatabaseSchema> {
@ -452,6 +467,21 @@ public interface CollectionDAO {
default String getNameHashColumn() {
return "fqnHash";
}
@ConnectionAwareSqlQuery(
value =
"select JSON_EXTRACT(json, '$.fullyQualifiedName') from database_schema_entity where id not in (select toId from entity_relationship where fromEntity = 'database' and toEntity = 'databaseSchema')",
connectionType = MYSQL)
@ConnectionAwareSqlQuery(
value =
"select json ->> 'fullyQualifiedName' from database_schema_entity where id not in (select toId from entity_relationship where fromEntity = 'database' and toEntity = 'databaseSchema')",
connectionType = POSTGRES)
List<String> getBrokenDatabaseSchemas();
@SqlUpdate(
value =
"delete from database_schema_entity where id not in (select toId from entity_relationship where fromEntity = 'database' and toEntity = 'databaseSchema')")
int removeBrokenDatabaseSchemas();
}
interface DatabaseServiceDAO extends EntityDAO<DatabaseService> {
@ -3007,6 +3037,21 @@ public interface CollectionDAO {
return "fqnHash";
}
@ConnectionAwareSqlQuery(
value =
"select JSON_EXTRACT(json, '$.fullyQualifiedName') from table_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseSchema' and toEntity = 'table')",
connectionType = MYSQL)
@ConnectionAwareSqlQuery(
value =
"select json ->> 'fullyQualifiedName' from table_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseSchema' and toEntity = 'table')",
connectionType = POSTGRES)
List<String> getBrokenTables();
@SqlUpdate(
value =
"delete from table_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseSchema' and toEntity = 'table')")
int removeBrokenTables();
@Override
default int listCount(ListFilter filter) {
String includeEmptyTestSuite = filter.getQueryParam("includeEmptyTestSuite");

62
openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java
View File

@ -646,6 +646,68 @@ public class OpenMetadataOperations implements Callable<Integer> {
}
}
@Command(
name = "dbServiceCleanup",
description = "Cleans Up broken relationship hierarchy for database service.")
public Integer cleanupOrphanedEntities() {
try {
LOG.info("Running a Database Service Hierarchy Cleanup");
parseConfig();
// Check Broken Tables
List<String> brokenTables = Entity.getCollectionDAO().tableDAO().getBrokenTables();
LOG.info("Following Tables seems to be Broken.");
List<String> tableColumns = List.of(String.format("Tables(%d)", brokenTables.size()));
List<List<String>> allRowsForTables = new ArrayList<>();
for (String name : brokenTables) {
List<String> row = new ArrayList<>();
row.add(name);
allRowsForTables.add(row);
}
printToAsciiTable(tableColumns.stream().toList(), allRowsForTables, "No Broken Tables.");
LOG.info("Cleaning up the broken tables.");
if (!brokenTables.isEmpty()) {
Entity.getCollectionDAO().tableDAO().removeBrokenTables();
}
List<String> brokenSchemas =
Entity.getCollectionDAO().databaseSchemaDAO().getBrokenDatabaseSchemas();
LOG.info("Following DatabaseSchemas seems to be Broken.");
List<String> dbSchemaColumns =
List.of(String.format("DatabaseSchemas(%d)", brokenSchemas.size()));
List<List<String>> allRowsForSchemas = new ArrayList<>();
for (String name : brokenSchemas) {
List<String> row = new ArrayList<>();
row.add(name);
allRowsForSchemas.add(row);
}
printToAsciiTable(dbSchemaColumns.stream().toList(), allRowsForSchemas, "No Broken Schemas.");
if (!brokenSchemas.isEmpty()) {
Entity.getCollectionDAO().databaseSchemaDAO().removeBrokenDatabaseSchemas();
}
List<String> brokenDatabases = Entity.getCollectionDAO().databaseDAO().getBrokenDatabase();
LOG.info("Following Database seems to be Broken.");
List<String> databaseColumns = List.of(String.format("Database(%d)", brokenSchemas.size()));
List<List<String>> allRowsForDatabases = new ArrayList<>();
for (String name : brokenDatabases) {
List<String> row = new ArrayList<>();
row.add(name);
allRowsForDatabases.add(row);
}
printToAsciiTable(
databaseColumns.stream().toList(), allRowsForDatabases, "No Broken Databases.");
if (!brokenDatabases.isEmpty()) {
Entity.getCollectionDAO().databaseDAO().removeDatabase();
}
return 0;
} catch (Exception e) {
LOG.error("Failed to Entity Cleanup due to ", e);
return 1;
}
}
@Command(name = "reindex", description = "Re Indexes data into search engine from command line.")
public Integer reIndex(
@Option(