mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2025-06-27 04:22:05 +00:00
* Fix #21533 - Add a tool to openmetadata-ops.sh to delete orphaned relations * Fix #21533 - Add a tool to openmetadata-ops.sh to delete orphaned relations * Add exit 1 and make dry-run default * Add exit 1 and make dry-run default * Fix checkstyle --------- Co-authored-by: Pere Miquel Brull <peremiquelbrull@gmail.com>
This commit is contained in:
Sriharsha Chintalapani
GitHub
parent
5fc2c90e5b
commit
1c5772d6f8
@ -1351,6 +1351,15 @@ public interface CollectionDAO {
|
||||
List<EntityRelationshipObject> getRecordWithOffset(
|
||||
@Bind("relation") int relation, @Bind("offset") long offset, @Bind("limit") int limit);
|
||||
|
||||
@SqlQuery(
|
||||
"SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship ORDER BY fromId, toId LIMIT :limit OFFSET :offset")
|
||||
@RegisterRowMapper(RelationshipObjectMapper.class)
|
||||
List<EntityRelationshipObject> getAllRelationshipsPaginated(
|
||||
@Bind("offset") long offset, @Bind("limit") int limit);
|
||||
|
||||
@SqlQuery("SELECT COUNT(*) FROM entity_relationship")
|
||||
long getTotalRelationshipCount();
|
||||
|
||||
//
|
||||
// Delete Operations
|
||||
//
|
||||
|
||||
@ -0,0 +1,374 @@
|
||||
/*
|
||||
* Copyright 2021 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.openmetadata.service.util;
|
||||
|
||||
import static org.openmetadata.service.util.OpenMetadataOperations.printToAsciiTable;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.openmetadata.service.Entity;
|
||||
import org.openmetadata.service.exception.EntityNotFoundException;
|
||||
import org.openmetadata.service.jdbi3.CollectionDAO;
|
||||
import org.openmetadata.service.jdbi3.EntityRepository;
|
||||
|
||||
@Slf4j
|
||||
public class EntityRelationshipCleanup {
|
||||
|
||||
private final CollectionDAO collectionDAO;
|
||||
private final Map<String, EntityRepository<?>> entityRepositories;
|
||||
private final boolean dryRun;
|
||||
|
||||
public EntityRelationshipCleanup(CollectionDAO collectionDAO, boolean dryRun) {
|
||||
this.collectionDAO = collectionDAO;
|
||||
this.dryRun = dryRun;
|
||||
this.entityRepositories = new HashMap<>();
|
||||
initializeEntityRepositories();
|
||||
}
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static class OrphanedRelationship {
|
||||
private String fromId;
|
||||
private String toId;
|
||||
private String fromEntity;
|
||||
private String toEntity;
|
||||
private int relation;
|
||||
private String reason;
|
||||
private String relationshipName;
|
||||
}
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public static class CleanupResult {
|
||||
private int totalRelationshipsScanned;
|
||||
private int orphanedRelationshipsFound;
|
||||
private int relationshipsDeleted;
|
||||
private List<OrphanedRelationship> orphanedRelationships;
|
||||
private Map<String, Integer> orphansByEntityType;
|
||||
private Map<String, Integer> orphansByRelationType;
|
||||
}
|
||||
|
||||
private void initializeEntityRepositories() {
|
||||
for (String entityType : Entity.getEntityList()) {
|
||||
try {
|
||||
EntityRepository<?> repository = Entity.getEntityRepository(entityType);
|
||||
entityRepositories.put(entityType, repository);
|
||||
} catch (EntityNotFoundException e) {
|
||||
LOG.error("No repository found for entity type: {}", entityType);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public CleanupResult performCleanup(int batchSize) {
|
||||
LOG.info(
|
||||
"Starting entity relationship cleanup. Dry run: {}, Batch size: {}", dryRun, batchSize);
|
||||
|
||||
CleanupResult result =
|
||||
CleanupResult.builder()
|
||||
.orphanedRelationships(new ArrayList<>())
|
||||
.orphansByEntityType(new HashMap<>())
|
||||
.orphansByRelationType(new HashMap<>())
|
||||
.build();
|
||||
|
||||
try {
|
||||
long totalRelationships = collectionDAO.relationshipDAO().getTotalRelationshipCount();
|
||||
result.setTotalRelationshipsScanned((int) totalRelationships);
|
||||
|
||||
LOG.info(
|
||||
"Found {} total relationships to scan. Processing in batches of {}",
|
||||
totalRelationships,
|
||||
batchSize);
|
||||
|
||||
long offset = 0;
|
||||
int processedCount = 0;
|
||||
int batchNumber = 1;
|
||||
|
||||
while (offset < totalRelationships) {
|
||||
LOG.info("Processing batch {} (offset: {}, limit: {})", batchNumber, offset, batchSize);
|
||||
|
||||
List<CollectionDAO.EntityRelationshipObject> relationshipBatch =
|
||||
collectionDAO.relationshipDAO().getAllRelationshipsPaginated(offset, batchSize);
|
||||
|
||||
if (relationshipBatch.isEmpty()) {
|
||||
LOG.info("No more relationships to process");
|
||||
break;
|
||||
}
|
||||
|
||||
for (CollectionDAO.EntityRelationshipObject relationship : relationshipBatch) {
|
||||
OrphanedRelationship orphan = validateRelationship(relationship);
|
||||
if (orphan != null) {
|
||||
result.getOrphanedRelationships().add(orphan);
|
||||
|
||||
result
|
||||
.getOrphansByEntityType()
|
||||
.merge(orphan.getFromEntity() + "->" + orphan.getToEntity(), 1, Integer::sum);
|
||||
result.getOrphansByRelationType().merge(orphan.getRelationshipName(), 1, Integer::sum);
|
||||
}
|
||||
processedCount++;
|
||||
}
|
||||
|
||||
offset += relationshipBatch.size();
|
||||
batchNumber++;
|
||||
|
||||
if (processedCount % (batchSize * 10) == 0 || offset >= totalRelationships) {
|
||||
LOG.info(
|
||||
"Progress: {}/{} relationships processed, {} orphaned relationships found",
|
||||
processedCount,
|
||||
totalRelationships,
|
||||
result.getOrphanedRelationships().size());
|
||||
}
|
||||
}
|
||||
|
||||
result.setOrphanedRelationshipsFound(result.getOrphanedRelationships().size());
|
||||
|
||||
LOG.info(
|
||||
"Completed scanning {} relationships. Found {} orphaned relationships",
|
||||
processedCount,
|
||||
result.getOrphanedRelationshipsFound());
|
||||
|
||||
displayOrphanedRelationships(result);
|
||||
if (!dryRun && !result.getOrphanedRelationships().isEmpty()) {
|
||||
result.setRelationshipsDeleted(
|
||||
deleteOrphanedRelationships(result.getOrphanedRelationships()));
|
||||
}
|
||||
|
||||
LOG.info(
|
||||
"Entity relationship cleanup completed. Scanned: {}, Found: {}, Deleted: {}",
|
||||
processedCount,
|
||||
result.getOrphanedRelationshipsFound(),
|
||||
result.getRelationshipsDeleted());
|
||||
|
||||
} catch (Exception e) {
|
||||
LOG.error("Error during entity relationship cleanup", e);
|
||||
throw new RuntimeException("Entity relationship cleanup failed", e);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private OrphanedRelationship validateRelationship(
|
||||
CollectionDAO.EntityRelationshipObject relationship) {
|
||||
try {
|
||||
UUID fromId = UUID.fromString(relationship.getFromId());
|
||||
UUID toId = UUID.fromString(relationship.getToId());
|
||||
String fromEntity = relationship.getFromEntity();
|
||||
String toEntity = relationship.getToEntity();
|
||||
boolean fromExists = entityExists(fromId, fromEntity);
|
||||
boolean toExists = entityExists(toId, toEntity);
|
||||
|
||||
if (fromExists && toExists) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String reason;
|
||||
if (!fromExists && !toExists) {
|
||||
reason = "Both fromEntity and toEntity do not exist";
|
||||
} else if (!fromExists) {
|
||||
reason = "fromEntity does not exist";
|
||||
} else {
|
||||
reason = "toEntity does not exist";
|
||||
}
|
||||
|
||||
return OrphanedRelationship.builder()
|
||||
.fromId(relationship.getFromId())
|
||||
.toId(relationship.getToId())
|
||||
.fromEntity(fromEntity)
|
||||
.toEntity(toEntity)
|
||||
.relation(relationship.getRelation())
|
||||
.reason(reason)
|
||||
.relationshipName(getRelationshipName(relationship.getRelation()))
|
||||
.build();
|
||||
|
||||
} catch (Exception e) {
|
||||
LOG.debug(
|
||||
"Error validating relationship {}->{}: {}",
|
||||
relationship.getFromId(),
|
||||
relationship.getToId(),
|
||||
e.getMessage());
|
||||
|
||||
return OrphanedRelationship.builder()
|
||||
.fromId(relationship.getFromId())
|
||||
.toId(relationship.getToId())
|
||||
.fromEntity(relationship.getFromEntity())
|
||||
.toEntity(relationship.getToEntity())
|
||||
.relation(relationship.getRelation())
|
||||
.reason("Validation error: " + e.getMessage())
|
||||
.relationshipName(getRelationshipName(relationship.getRelation()))
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean entityExists(UUID entityId, String entityType) {
|
||||
try {
|
||||
EntityRepository<?> repository = entityRepositories.get(entityType);
|
||||
if (repository == null) {
|
||||
LOG.debug("No repository found for entity type: {}", entityType);
|
||||
return false;
|
||||
}
|
||||
repository.get(null, entityId, EntityUtil.Fields.EMPTY_FIELDS);
|
||||
return true;
|
||||
} catch (EntityNotFoundException e) {
|
||||
LOG.debug("Entity {}:{} not found", entityType, entityId);
|
||||
return false;
|
||||
} catch (Exception e) {
|
||||
LOG.debug(
|
||||
"Error checking existence of entity {}:{}: {}", entityType, entityId, e.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes orphaned relationships from the database
|
||||
*
|
||||
* @param orphanedRelationships List of orphaned relationships to delete
|
||||
* @return Number of relationships successfully deleted
|
||||
*/
|
||||
private int deleteOrphanedRelationships(List<OrphanedRelationship> orphanedRelationships) {
|
||||
LOG.info("Deleting {} orphaned relationships", orphanedRelationships.size());
|
||||
int deletedCount = 0;
|
||||
|
||||
for (OrphanedRelationship orphan : orphanedRelationships) {
|
||||
try {
|
||||
UUID fromId = UUID.fromString(orphan.getFromId());
|
||||
UUID toId = UUID.fromString(orphan.getToId());
|
||||
|
||||
int deleted =
|
||||
collectionDAO
|
||||
.relationshipDAO()
|
||||
.delete(
|
||||
fromId,
|
||||
orphan.getFromEntity(),
|
||||
toId,
|
||||
orphan.getToEntity(),
|
||||
orphan.getRelation());
|
||||
|
||||
if (deleted > 0) {
|
||||
deletedCount++;
|
||||
LOG.debug(
|
||||
"Deleted orphaned relationship: {} {} -> {} {}",
|
||||
orphan.getFromEntity(),
|
||||
orphan.getFromId(),
|
||||
orphan.getToEntity(),
|
||||
orphan.getToId());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOG.error(
|
||||
"Failed to delete orphaned relationship: {} {} -> {} {}: {}",
|
||||
orphan.getFromEntity(),
|
||||
orphan.getFromId(),
|
||||
orphan.getToEntity(),
|
||||
orphan.getToId(),
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
LOG.info(
|
||||
"Successfully deleted {} out of {} orphaned relationships",
|
||||
deletedCount,
|
||||
orphanedRelationships.size());
|
||||
return deletedCount;
|
||||
}
|
||||
|
||||
private void displayOrphanedRelationships(CleanupResult result) {
|
||||
if (result.getOrphanedRelationships().isEmpty()) {
|
||||
LOG.info("No orphaned relationships found. All entity relationships are valid.");
|
||||
return;
|
||||
}
|
||||
|
||||
LOG.info("Found {} orphaned relationships", result.getOrphanedRelationshipsFound());
|
||||
|
||||
// Display detailed table of orphaned relationships
|
||||
List<String> columns =
|
||||
Arrays.asList("From Entity", "From ID", "To Entity", "To ID", "Relation", "Reason");
|
||||
|
||||
List<List<String>> rows = new ArrayList<>();
|
||||
for (OrphanedRelationship orphan : result.getOrphanedRelationships()) {
|
||||
rows.add(
|
||||
Arrays.asList(
|
||||
orphan.getFromEntity(),
|
||||
orphan.getFromId(),
|
||||
orphan.getToEntity(),
|
||||
orphan.getToId(),
|
||||
orphan.getRelationshipName(),
|
||||
orphan.getReason()));
|
||||
}
|
||||
|
||||
printToAsciiTable(columns, rows, "No orphaned relationships found");
|
||||
|
||||
// Display summary statistics
|
||||
displaySummaryStatistics(result);
|
||||
}
|
||||
|
||||
private void displaySummaryStatistics(CleanupResult result) {
|
||||
if (!result.getOrphansByEntityType().isEmpty()) {
|
||||
LOG.info("Orphaned relationships by entity type:");
|
||||
List<String> entityColumns = Arrays.asList("Entity Type Pair", "Count");
|
||||
List<List<String>> entityRows = new ArrayList<>();
|
||||
|
||||
result.getOrphansByEntityType().entrySet().stream()
|
||||
.sorted(Map.Entry.<String, Integer>comparingByValue().reversed())
|
||||
.forEach(
|
||||
entry -> entityRows.add(Arrays.asList(entry.getKey(), entry.getValue().toString())));
|
||||
|
||||
printToAsciiTable(entityColumns, entityRows, "No entity type statistics");
|
||||
}
|
||||
|
||||
if (!result.getOrphansByRelationType().isEmpty()) {
|
||||
LOG.info("Orphaned relationships by relation type:");
|
||||
List<String> relationColumns = Arrays.asList("Relation Type", "Count");
|
||||
List<List<String>> relationRows = new ArrayList<>();
|
||||
|
||||
result.getOrphansByRelationType().entrySet().stream()
|
||||
.sorted(Map.Entry.<String, Integer>comparingByValue().reversed())
|
||||
.forEach(
|
||||
entry ->
|
||||
relationRows.add(Arrays.asList(entry.getKey(), entry.getValue().toString())));
|
||||
|
||||
printToAsciiTable(relationColumns, relationRows, "No relation type statistics");
|
||||
}
|
||||
}
|
||||
|
||||
private String getRelationshipName(int relation) {
|
||||
// Map common relationship types to names
|
||||
// These constants should ideally be imported from the actual schema
|
||||
return switch (relation) {
|
||||
case 10 -> "CONTAINS";
|
||||
case 11 -> "CREATED_BY";
|
||||
case 12 -> "MENTIONED_IN";
|
||||
case 13 -> "PARENT_OF";
|
||||
case 14 -> "OWNS";
|
||||
case 15 -> "FOLLOWS";
|
||||
case 16 -> "JOINED";
|
||||
case 17 -> "REACTED_TO";
|
||||
case 18 -> "REPLIED_TO";
|
||||
case 19 -> "TESTED_BY";
|
||||
case 20 -> "UPSTREAM";
|
||||
case 21 -> "DOWNSTREAM";
|
||||
default -> "RELATION_" + relation;
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -140,7 +140,8 @@ public class OpenMetadataOperations implements Callable<Integer> {
|
||||
public Integer call() {
|
||||
LOG.info(
|
||||
"Subcommand needed: 'info', 'validate', 'repair', 'check-connection', "
|
||||
+ "'drop-create', 'changelog', 'migrate', 'migrate-secrets', 'reindex', 'deploy-pipelines'");
|
||||
+ "'drop-create', 'changelog', 'migrate', 'migrate-secrets', 'reindex', 'deploy-pipelines', "
|
||||
+ "'dbServiceCleanup', 'relationshipCleanup'");
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -709,6 +710,48 @@ public class OpenMetadataOperations implements Callable<Integer> {
|
||||
}
|
||||
}
|
||||
|
||||
@Command(
|
||||
name = "relationshipCleanup",
|
||||
description =
|
||||
"Cleans up orphaned entity relationships where referenced entities no longer exist. By default, runs in dry-run mode to only identify orphaned relationships.")
|
||||
public Integer cleanupOrphanedRelationships(
|
||||
@Option(
|
||||
names = {"--delete"},
|
||||
description =
|
||||
"Actually delete the orphaned relationships. Without this flag, the command only identifies orphaned relationships (dry-run mode).",
|
||||
defaultValue = "false")
|
||||
boolean delete,
|
||||
@Option(
|
||||
names = {"-b", "--batch-size"},
|
||||
defaultValue = "1000",
|
||||
description = "Number of relationships to process in each batch.")
|
||||
int batchSize) {
|
||||
try {
|
||||
boolean dryRun = !delete;
|
||||
LOG.info(
|
||||
"Running Entity Relationship Cleanup. Dry run: {}, Batch size: {}", dryRun, batchSize);
|
||||
parseConfig();
|
||||
|
||||
EntityRelationshipCleanup cleanup = new EntityRelationshipCleanup(collectionDAO, dryRun);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(batchSize);
|
||||
|
||||
LOG.info("=== Entity Relationship Cleanup Summary ===");
|
||||
LOG.info("Total relationships scanned: {}", result.getTotalRelationshipsScanned());
|
||||
LOG.info("Orphaned relationships found: {}", result.getOrphanedRelationshipsFound());
|
||||
LOG.info("Relationships deleted: {}", result.getRelationshipsDeleted());
|
||||
|
||||
if (dryRun && result.getOrphanedRelationshipsFound() > 0) {
|
||||
LOG.info("To actually delete these orphaned relationships, run with --delete");
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
} catch (Exception e) {
|
||||
LOG.error("Failed to cleanup orphaned relationships due to ", e);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
@Command(name = "reindex", description = "Re Indexes data into search engine from command line.")
|
||||
public Integer reIndex(
|
||||
@Option(
|
||||
|
||||
@ -0,0 +1,473 @@
|
||||
/*
|
||||
* Copyright 2025 Collate
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.openmetadata.service.util;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInfo;
|
||||
import org.junit.jupiter.api.parallel.Execution;
|
||||
import org.junit.jupiter.api.parallel.ExecutionMode;
|
||||
import org.openmetadata.schema.api.data.CreateTable;
|
||||
import org.openmetadata.schema.entity.data.Database;
|
||||
import org.openmetadata.schema.entity.data.DatabaseSchema;
|
||||
import org.openmetadata.schema.entity.data.Table;
|
||||
import org.openmetadata.schema.type.Relationship;
|
||||
import org.openmetadata.service.Entity;
|
||||
import org.openmetadata.service.OpenMetadataApplicationTest;
|
||||
import org.openmetadata.service.jdbi3.CollectionDAO;
|
||||
import org.openmetadata.service.resources.EntityResourceTest;
|
||||
import org.openmetadata.service.resources.databases.TableResourceTest;
|
||||
import org.openmetadata.service.resources.services.DatabaseServiceResourceTest;
|
||||
|
||||
@Slf4j
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
class EntityRelationshipCleanupTest extends OpenMetadataApplicationTest {
|
||||
|
||||
private static TableResourceTest tableTest;
|
||||
|
||||
private static List<Table> testTables;
|
||||
|
||||
private static CollectionDAO collectionDAO;
|
||||
private static EntityRelationshipCleanup cleanup;
|
||||
|
||||
@BeforeAll
|
||||
static void setup(TestInfo test) throws IOException {
|
||||
DatabaseServiceResourceTest serviceTest = new DatabaseServiceResourceTest();
|
||||
tableTest = new TableResourceTest();
|
||||
testTables = new ArrayList<>();
|
||||
|
||||
collectionDAO = Entity.getCollectionDAO();
|
||||
serviceTest.setupDatabaseServices(test);
|
||||
tableTest.setupDatabaseSchemas(test);
|
||||
setupTestEntities(test);
|
||||
}
|
||||
|
||||
private static void setupTestEntities(TestInfo test) throws IOException {
|
||||
Database testDatabase = EntityResourceTest.DATABASE;
|
||||
DatabaseSchema testSchema = EntityResourceTest.DATABASE_SCHEMA;
|
||||
|
||||
for (int i = 0; i < 3; i++) {
|
||||
CreateTable createTable =
|
||||
tableTest.createRequest(test, i).withDatabaseSchema(testSchema.getFullyQualifiedName());
|
||||
Table table = tableTest.createEntity(createTable, ADMIN_AUTH_HEADERS);
|
||||
testTables.add(table);
|
||||
}
|
||||
|
||||
LOG.info(
|
||||
"Created test entities: Database={}, Schema={}, Tables={}",
|
||||
testDatabase.getId(),
|
||||
testSchema.getId(),
|
||||
testTables.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_cleanupWithValidRelationships_shouldFindNoOrphans() {
|
||||
cleanup = new EntityRelationshipCleanup(collectionDAO, true); // dry-run mode
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(100);
|
||||
assertNotNull(result);
|
||||
assertTrue(result.getTotalRelationshipsScanned() >= 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_cleanupAfterDeletingEntity_shouldDetectOrphans() {
|
||||
Table tableToDelete = testTables.get(0);
|
||||
UUID tableId = tableToDelete.getId();
|
||||
collectionDAO.tableDAO().delete(tableId);
|
||||
cleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(100);
|
||||
assertNotNull(result);
|
||||
assertTrue(
|
||||
result.getOrphanedRelationshipsFound() > 0,
|
||||
"Should find orphaned relationships after entity deletion");
|
||||
boolean foundOrphanedTable =
|
||||
result.getOrphanedRelationships().stream()
|
||||
.anyMatch(
|
||||
orphan ->
|
||||
tableId.toString().equals(orphan.getFromId())
|
||||
|| tableId.toString().equals(orphan.getToId()));
|
||||
assertTrue(foundOrphanedTable, "Should find orphaned relationships for deleted table");
|
||||
|
||||
assertFalse(result.getOrphansByEntityType().isEmpty(), "Should have entity type statistics");
|
||||
assertFalse(
|
||||
result.getOrphansByRelationType().isEmpty(), "Should have relation type statistics");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_actualCleanup_shouldDeleteOrphanedRelationships() {
|
||||
EntityRelationshipCleanup dryRunCleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult dryRunResult = dryRunCleanup.performCleanup(100);
|
||||
|
||||
int orphansFoundInDryRun = dryRunResult.getOrphanedRelationshipsFound();
|
||||
|
||||
if (orphansFoundInDryRun > 0) {
|
||||
EntityRelationshipCleanup actualCleanup = new EntityRelationshipCleanup(collectionDAO, false);
|
||||
EntityRelationshipCleanup.CleanupResult actualResult = actualCleanup.performCleanup(100);
|
||||
|
||||
assertNotNull(actualResult);
|
||||
assertTrue(
|
||||
actualResult.getRelationshipsDeleted() > 0, "Should have deleted orphaned relationships");
|
||||
assertEquals(
|
||||
orphansFoundInDryRun,
|
||||
actualResult.getRelationshipsDeleted(),
|
||||
"Should delete same number of relationships as found in dry run");
|
||||
|
||||
EntityRelationshipCleanup verificationCleanup =
|
||||
new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult verificationResult =
|
||||
verificationCleanup.performCleanup(100);
|
||||
|
||||
assertEquals(
|
||||
0,
|
||||
verificationResult.getOrphanedRelationshipsFound(),
|
||||
"Should find no orphaned relationships after cleanup");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_paginationWithLargeBatchSize() {
|
||||
cleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(10000);
|
||||
assertNotNull(result);
|
||||
assertTrue(result.getTotalRelationshipsScanned() >= 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_paginationWithSmallBatchSize() {
|
||||
cleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(10);
|
||||
assertNotNull(result);
|
||||
assertTrue(result.getTotalRelationshipsScanned() >= 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_createOrphanedRelationshipScenario() {
|
||||
UUID nonExistentId1 = UUID.randomUUID();
|
||||
UUID nonExistentId2 = UUID.randomUUID();
|
||||
|
||||
collectionDAO
|
||||
.relationshipDAO()
|
||||
.insert(
|
||||
nonExistentId1,
|
||||
nonExistentId2,
|
||||
"table",
|
||||
"databaseSchema",
|
||||
Relationship.CONTAINS.ordinal(),
|
||||
null);
|
||||
cleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(100);
|
||||
assertTrue(
|
||||
result.getOrphanedRelationshipsFound() > 0,
|
||||
"Should detect manually created orphaned relationship");
|
||||
|
||||
boolean foundSpecificOrphan =
|
||||
result.getOrphanedRelationships().stream()
|
||||
.anyMatch(
|
||||
orphan ->
|
||||
nonExistentId1.toString().equals(orphan.getFromId())
|
||||
&& nonExistentId2.toString().equals(orphan.getToId()));
|
||||
|
||||
assertTrue(foundSpecificOrphan, "Should find the specific orphaned relationship created");
|
||||
|
||||
EntityRelationshipCleanup actualCleanup = new EntityRelationshipCleanup(collectionDAO, false);
|
||||
EntityRelationshipCleanup.CleanupResult cleanupResult = actualCleanup.performCleanup(100);
|
||||
|
||||
assertTrue(
|
||||
cleanupResult.getRelationshipsDeleted() > 0, "Should delete the orphaned relationship");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_validationOfExistingRelationships() {
|
||||
long relationshipCountBefore = collectionDAO.relationshipDAO().getTotalRelationshipCount();
|
||||
cleanup = new EntityRelationshipCleanup(collectionDAO, false);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(100);
|
||||
long relationshipCountAfter = collectionDAO.relationshipDAO().getTotalRelationshipCount();
|
||||
long expectedCount = relationshipCountBefore - result.getRelationshipsDeleted();
|
||||
assertEquals(
|
||||
expectedCount,
|
||||
relationshipCountAfter,
|
||||
"Relationship count should match expected after cleanup");
|
||||
|
||||
LOG.info(
|
||||
"Validation test - Before: {}, After: {}, Deleted: {}",
|
||||
relationshipCountBefore,
|
||||
relationshipCountAfter,
|
||||
result.getRelationshipsDeleted());
|
||||
|
||||
for (Table table : testTables.subList(1, testTables.size())) {
|
||||
try {
|
||||
Table retrievedTable = tableTest.getEntity(table.getId(), ADMIN_AUTH_HEADERS);
|
||||
assertNotNull(retrievedTable, "Valid table should still exist");
|
||||
assertNotNull(
|
||||
retrievedTable.getDatabaseSchema(), "Table should still have schema reference");
|
||||
} catch (Exception e) {
|
||||
// If table was part of orphaned cleanup, that's expected
|
||||
LOG.info("Table {} not found after cleanup (expected if it was orphaned)", table.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_relationshipCleanupCommand_dryRun() {
|
||||
EntityRelationshipCleanup dryRunCleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult result = dryRunCleanup.performCleanup(500);
|
||||
assertNotNull(result, "Cleanup result should not be null");
|
||||
assertTrue(result.getTotalRelationshipsScanned() >= 0, "Should scan some relationships");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_relationshipCleanupCommand_withOrphanedData() {
|
||||
UUID nonExistentEntityId = UUID.randomUUID();
|
||||
|
||||
collectionDAO
|
||||
.relationshipDAO()
|
||||
.insert(
|
||||
testTables.get(1).getId(),
|
||||
nonExistentEntityId,
|
||||
"table",
|
||||
"databaseSchema",
|
||||
Relationship.CONTAINS.ordinal(),
|
||||
null);
|
||||
|
||||
EntityRelationshipCleanup dryRunCleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult dryRunResult = dryRunCleanup.performCleanup(100);
|
||||
|
||||
assertNotNull(dryRunResult, "Dry-run result should not be null");
|
||||
assertTrue(dryRunResult.getOrphanedRelationshipsFound() > 0, "Should find orphaned data");
|
||||
|
||||
EntityRelationshipCleanup actualCleanup = new EntityRelationshipCleanup(collectionDAO, false);
|
||||
EntityRelationshipCleanup.CleanupResult cleanupResult = actualCleanup.performCleanup(100);
|
||||
|
||||
assertNotNull(cleanupResult, "Cleanup result should not be null");
|
||||
assertTrue(cleanupResult.getRelationshipsDeleted() > 0, "Should delete orphaned relationships");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_relationshipCleanupCommand_noOrphanedData() {
|
||||
EntityRelationshipCleanup cleanup1 = new EntityRelationshipCleanup(collectionDAO, false);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup1.performCleanup(100);
|
||||
assertNotNull(result, "Cleanup result should not be null");
|
||||
assertEquals(
|
||||
0,
|
||||
result.getRelationshipsDeleted(),
|
||||
"Should not delete any relationships when none are orphaned");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_relationshipCleanupCommand_smallBatchSize() {
|
||||
EntityRelationshipCleanup cleanup1 = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup1.performCleanup(10);
|
||||
|
||||
assertNotNull(result, "Cleanup result should not be null");
|
||||
assertTrue(result.getTotalRelationshipsScanned() >= 0, "Should scan relationships");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_relationshipCleanupCommand_largeBatchSize() {
|
||||
EntityRelationshipCleanup cleanup1 = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup1.performCleanup(10000);
|
||||
|
||||
assertNotNull(result, "Cleanup result should not be null");
|
||||
assertTrue(result.getTotalRelationshipsScanned() >= 0, "Should scan relationships");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_relationshipCleanupCommand_multipleOrphanedRelationships() {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
UUID nonExistentId1 = UUID.randomUUID();
|
||||
UUID nonExistentId2 = UUID.randomUUID();
|
||||
|
||||
collectionDAO
|
||||
.relationshipDAO()
|
||||
.insert(
|
||||
nonExistentId1,
|
||||
nonExistentId2,
|
||||
"table",
|
||||
"databaseSchema",
|
||||
Relationship.CONTAINS.ordinal(),
|
||||
null);
|
||||
}
|
||||
|
||||
EntityRelationshipCleanup cleanup1 = new EntityRelationshipCleanup(collectionDAO, false);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup1.performCleanup(2);
|
||||
|
||||
assertNotNull(result, "Cleanup result should not be null");
|
||||
assertTrue(
|
||||
result.getRelationshipsDeleted() >= 5,
|
||||
"Should delete at least the 5 orphaned relationships created");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_relationshipCleanupCommand_validationOfParameters() {
|
||||
|
||||
EntityRelationshipCleanup minBatchCleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult minBatchResult = minBatchCleanup.performCleanup(1);
|
||||
|
||||
assertNotNull(minBatchResult, "Minimum batch size result should not be null");
|
||||
EntityRelationshipCleanup defaultCleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult defaultResult = defaultCleanup.performCleanup(1000);
|
||||
|
||||
assertNotNull(defaultResult, "Default batch size result should not be null");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_commandIntegrationValidation() {
|
||||
EntityRelationshipCleanup cleanup1 = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup1.performCleanup(100);
|
||||
|
||||
assertNotNull(result, "Command integration result should not be null");
|
||||
assertTrue(result.getTotalRelationshipsScanned() >= 0, "Should scan relationships");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_emptyDatabaseScenario() {
|
||||
// Test cleanup behavior when there are minimal relationships
|
||||
|
||||
// Run cleanup on current database
|
||||
cleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(100);
|
||||
|
||||
assertNotNull(result);
|
||||
assertTrue(result.getTotalRelationshipsScanned() >= 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_commandBehavior_defaultDryRunWithNoOrphans() {
|
||||
EntityRelationshipCleanup dryRunCleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult result = dryRunCleanup.performCleanup(100);
|
||||
|
||||
assertNotNull(result);
|
||||
|
||||
boolean wouldReturnOne = result.getOrphanedRelationshipsFound() > 0;
|
||||
assertFalse(
|
||||
wouldReturnOne || result.getOrphanedRelationshipsFound() < 0,
|
||||
"Default dry-run with no orphans should indicate success (exit code 0)");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_commandBehavior_defaultDryRunWithOrphans() {
|
||||
UUID nonExistentId1 = UUID.randomUUID();
|
||||
UUID nonExistentId2 = UUID.randomUUID();
|
||||
|
||||
collectionDAO
|
||||
.relationshipDAO()
|
||||
.insert(
|
||||
nonExistentId1,
|
||||
nonExistentId2,
|
||||
"table",
|
||||
"databaseSchema",
|
||||
Relationship.CONTAINS.ordinal(),
|
||||
null);
|
||||
|
||||
EntityRelationshipCleanup dryRunCleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult result = dryRunCleanup.performCleanup(100);
|
||||
|
||||
assertNotNull(result);
|
||||
|
||||
boolean wouldReturnOne = result.getOrphanedRelationshipsFound() > 0;
|
||||
assertTrue(
|
||||
wouldReturnOne, "Default dry-run with orphans found should indicate failure (exit code 1)");
|
||||
|
||||
assertTrue(result.getOrphanedRelationshipsFound() > 0, "Should find the orphaned relationship");
|
||||
assertEquals(0, result.getRelationshipsDeleted(), "Should not delete in dry-run mode");
|
||||
|
||||
EntityRelationshipCleanup actualCleanup = new EntityRelationshipCleanup(collectionDAO, false);
|
||||
actualCleanup.performCleanup(100);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_commandBehavior_explicitDeleteFlag() {
|
||||
UUID nonExistentId1 = UUID.randomUUID();
|
||||
UUID nonExistentId2 = UUID.randomUUID();
|
||||
|
||||
collectionDAO
|
||||
.relationshipDAO()
|
||||
.insert(
|
||||
nonExistentId1,
|
||||
nonExistentId2,
|
||||
"table",
|
||||
"databaseSchema",
|
||||
Relationship.CONTAINS.ordinal(),
|
||||
null);
|
||||
|
||||
EntityRelationshipCleanup deleteCleanup = new EntityRelationshipCleanup(collectionDAO, false);
|
||||
EntityRelationshipCleanup.CleanupResult result = deleteCleanup.performCleanup(100);
|
||||
|
||||
assertNotNull(result);
|
||||
assertTrue(result.getRelationshipsDeleted() > 0, "Should have deleted orphaned relationships");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_commandBehavior_batchSizeParameter() {
|
||||
int customBatchSize = 50;
|
||||
|
||||
EntityRelationshipCleanup cleanup = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(customBatchSize);
|
||||
|
||||
assertNotNull(result);
|
||||
assertTrue(result.getTotalRelationshipsScanned() >= 0);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Execution(ExecutionMode.CONCURRENT)
|
||||
void test_commandBehavior_flagSemantics() {
|
||||
EntityRelationshipCleanup defaultBehavior = new EntityRelationshipCleanup(collectionDAO, true);
|
||||
EntityRelationshipCleanup.CleanupResult defaultResult = defaultBehavior.performCleanup(100);
|
||||
|
||||
EntityRelationshipCleanup deleteBehavior = new EntityRelationshipCleanup(collectionDAO, false);
|
||||
EntityRelationshipCleanup.CleanupResult deleteResult = deleteBehavior.performCleanup(100);
|
||||
|
||||
assertNotNull(defaultResult);
|
||||
assertNotNull(deleteResult);
|
||||
|
||||
assertEquals(
|
||||
0,
|
||||
defaultResult.getRelationshipsDeleted(),
|
||||
"Default behavior (dry-run) should not delete any relationships");
|
||||
|
||||
assertTrue(
|
||||
deleteResult.getRelationshipsDeleted() >= 0,
|
||||
"Delete mode should delete 0 or more relationships");
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user