diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 6bee7b25ea4..e3b3a2d2983 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -1351,6 +1351,15 @@ public interface CollectionDAO { List getRecordWithOffset( @Bind("relation") int relation, @Bind("offset") long offset, @Bind("limit") int limit); + @SqlQuery( + "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship ORDER BY fromId, toId LIMIT :limit OFFSET :offset") + @RegisterRowMapper(RelationshipObjectMapper.class) + List getAllRelationshipsPaginated( + @Bind("offset") long offset, @Bind("limit") int limit); + + @SqlQuery("SELECT COUNT(*) FROM entity_relationship") + long getTotalRelationshipCount(); + // // Delete Operations // diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityRelationshipCleanup.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityRelationshipCleanup.java new file mode 100644 index 00000000000..93bf74121a4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityRelationshipCleanup.java @@ -0,0 +1,374 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.util; + +import static org.openmetadata.service.util.OpenMetadataOperations.printToAsciiTable; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.EntityRepository; + +@Slf4j +public class EntityRelationshipCleanup { + + private final CollectionDAO collectionDAO; + private final Map> entityRepositories; + private final boolean dryRun; + + public EntityRelationshipCleanup(CollectionDAO collectionDAO, boolean dryRun) { + this.collectionDAO = collectionDAO; + this.dryRun = dryRun; + this.entityRepositories = new HashMap<>(); + initializeEntityRepositories(); + } + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class OrphanedRelationship { + private String fromId; + private String toId; + private String fromEntity; + private String toEntity; + private int relation; + private String reason; + private String relationshipName; + } + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class CleanupResult { + private int totalRelationshipsScanned; + private int orphanedRelationshipsFound; + private int relationshipsDeleted; + private List orphanedRelationships; + private Map orphansByEntityType; + private Map orphansByRelationType; + } + + private void initializeEntityRepositories() { + for (String entityType : Entity.getEntityList()) { + try { + EntityRepository repository = Entity.getEntityRepository(entityType); + entityRepositories.put(entityType, repository); + } catch (EntityNotFoundException e) { + LOG.error("No repository found for entity type: {}", entityType); + } + } + } + + public CleanupResult performCleanup(int batchSize) { + LOG.info( + "Starting entity relationship cleanup. Dry run: {}, Batch size: {}", dryRun, batchSize); + + CleanupResult result = + CleanupResult.builder() + .orphanedRelationships(new ArrayList<>()) + .orphansByEntityType(new HashMap<>()) + .orphansByRelationType(new HashMap<>()) + .build(); + + try { + long totalRelationships = collectionDAO.relationshipDAO().getTotalRelationshipCount(); + result.setTotalRelationshipsScanned((int) totalRelationships); + + LOG.info( + "Found {} total relationships to scan. Processing in batches of {}", + totalRelationships, + batchSize); + + long offset = 0; + int processedCount = 0; + int batchNumber = 1; + + while (offset < totalRelationships) { + LOG.info("Processing batch {} (offset: {}, limit: {})", batchNumber, offset, batchSize); + + List relationshipBatch = + collectionDAO.relationshipDAO().getAllRelationshipsPaginated(offset, batchSize); + + if (relationshipBatch.isEmpty()) { + LOG.info("No more relationships to process"); + break; + } + + for (CollectionDAO.EntityRelationshipObject relationship : relationshipBatch) { + OrphanedRelationship orphan = validateRelationship(relationship); + if (orphan != null) { + result.getOrphanedRelationships().add(orphan); + + result + .getOrphansByEntityType() + .merge(orphan.getFromEntity() + "->" + orphan.getToEntity(), 1, Integer::sum); + result.getOrphansByRelationType().merge(orphan.getRelationshipName(), 1, Integer::sum); + } + processedCount++; + } + + offset += relationshipBatch.size(); + batchNumber++; + + if (processedCount % (batchSize * 10) == 0 || offset >= totalRelationships) { + LOG.info( + "Progress: {}/{} relationships processed, {} orphaned relationships found", + processedCount, + totalRelationships, + result.getOrphanedRelationships().size()); + } + } + + result.setOrphanedRelationshipsFound(result.getOrphanedRelationships().size()); + + LOG.info( + "Completed scanning {} relationships. Found {} orphaned relationships", + processedCount, + result.getOrphanedRelationshipsFound()); + + displayOrphanedRelationships(result); + if (!dryRun && !result.getOrphanedRelationships().isEmpty()) { + result.setRelationshipsDeleted( + deleteOrphanedRelationships(result.getOrphanedRelationships())); + } + + LOG.info( + "Entity relationship cleanup completed. Scanned: {}, Found: {}, Deleted: {}", + processedCount, + result.getOrphanedRelationshipsFound(), + result.getRelationshipsDeleted()); + + } catch (Exception e) { + LOG.error("Error during entity relationship cleanup", e); + throw new RuntimeException("Entity relationship cleanup failed", e); + } + + return result; + } + + private OrphanedRelationship validateRelationship( + CollectionDAO.EntityRelationshipObject relationship) { + try { + UUID fromId = UUID.fromString(relationship.getFromId()); + UUID toId = UUID.fromString(relationship.getToId()); + String fromEntity = relationship.getFromEntity(); + String toEntity = relationship.getToEntity(); + boolean fromExists = entityExists(fromId, fromEntity); + boolean toExists = entityExists(toId, toEntity); + + if (fromExists && toExists) { + return null; + } + + String reason; + if (!fromExists && !toExists) { + reason = "Both fromEntity and toEntity do not exist"; + } else if (!fromExists) { + reason = "fromEntity does not exist"; + } else { + reason = "toEntity does not exist"; + } + + return OrphanedRelationship.builder() + .fromId(relationship.getFromId()) + .toId(relationship.getToId()) + .fromEntity(fromEntity) + .toEntity(toEntity) + .relation(relationship.getRelation()) + .reason(reason) + .relationshipName(getRelationshipName(relationship.getRelation())) + .build(); + + } catch (Exception e) { + LOG.debug( + "Error validating relationship {}->{}: {}", + relationship.getFromId(), + relationship.getToId(), + e.getMessage()); + + return OrphanedRelationship.builder() + .fromId(relationship.getFromId()) + .toId(relationship.getToId()) + .fromEntity(relationship.getFromEntity()) + .toEntity(relationship.getToEntity()) + .relation(relationship.getRelation()) + .reason("Validation error: " + e.getMessage()) + .relationshipName(getRelationshipName(relationship.getRelation())) + .build(); + } + } + + private boolean entityExists(UUID entityId, String entityType) { + try { + EntityRepository repository = entityRepositories.get(entityType); + if (repository == null) { + LOG.debug("No repository found for entity type: {}", entityType); + return false; + } + repository.get(null, entityId, EntityUtil.Fields.EMPTY_FIELDS); + return true; + } catch (EntityNotFoundException e) { + LOG.debug("Entity {}:{} not found", entityType, entityId); + return false; + } catch (Exception e) { + LOG.debug( + "Error checking existence of entity {}:{}: {}", entityType, entityId, e.getMessage()); + return false; + } + } + + /** + * Deletes orphaned relationships from the database + * + * @param orphanedRelationships List of orphaned relationships to delete + * @return Number of relationships successfully deleted + */ + private int deleteOrphanedRelationships(List orphanedRelationships) { + LOG.info("Deleting {} orphaned relationships", orphanedRelationships.size()); + int deletedCount = 0; + + for (OrphanedRelationship orphan : orphanedRelationships) { + try { + UUID fromId = UUID.fromString(orphan.getFromId()); + UUID toId = UUID.fromString(orphan.getToId()); + + int deleted = + collectionDAO + .relationshipDAO() + .delete( + fromId, + orphan.getFromEntity(), + toId, + orphan.getToEntity(), + orphan.getRelation()); + + if (deleted > 0) { + deletedCount++; + LOG.debug( + "Deleted orphaned relationship: {} {} -> {} {}", + orphan.getFromEntity(), + orphan.getFromId(), + orphan.getToEntity(), + orphan.getToId()); + } + } catch (Exception e) { + LOG.error( + "Failed to delete orphaned relationship: {} {} -> {} {}: {}", + orphan.getFromEntity(), + orphan.getFromId(), + orphan.getToEntity(), + orphan.getToId(), + e.getMessage()); + } + } + + LOG.info( + "Successfully deleted {} out of {} orphaned relationships", + deletedCount, + orphanedRelationships.size()); + return deletedCount; + } + + private void displayOrphanedRelationships(CleanupResult result) { + if (result.getOrphanedRelationships().isEmpty()) { + LOG.info("No orphaned relationships found. All entity relationships are valid."); + return; + } + + LOG.info("Found {} orphaned relationships", result.getOrphanedRelationshipsFound()); + + // Display detailed table of orphaned relationships + List columns = + Arrays.asList("From Entity", "From ID", "To Entity", "To ID", "Relation", "Reason"); + + List> rows = new ArrayList<>(); + for (OrphanedRelationship orphan : result.getOrphanedRelationships()) { + rows.add( + Arrays.asList( + orphan.getFromEntity(), + orphan.getFromId(), + orphan.getToEntity(), + orphan.getToId(), + orphan.getRelationshipName(), + orphan.getReason())); + } + + printToAsciiTable(columns, rows, "No orphaned relationships found"); + + // Display summary statistics + displaySummaryStatistics(result); + } + + private void displaySummaryStatistics(CleanupResult result) { + if (!result.getOrphansByEntityType().isEmpty()) { + LOG.info("Orphaned relationships by entity type:"); + List entityColumns = Arrays.asList("Entity Type Pair", "Count"); + List> entityRows = new ArrayList<>(); + + result.getOrphansByEntityType().entrySet().stream() + .sorted(Map.Entry.comparingByValue().reversed()) + .forEach( + entry -> entityRows.add(Arrays.asList(entry.getKey(), entry.getValue().toString()))); + + printToAsciiTable(entityColumns, entityRows, "No entity type statistics"); + } + + if (!result.getOrphansByRelationType().isEmpty()) { + LOG.info("Orphaned relationships by relation type:"); + List relationColumns = Arrays.asList("Relation Type", "Count"); + List> relationRows = new ArrayList<>(); + + result.getOrphansByRelationType().entrySet().stream() + .sorted(Map.Entry.comparingByValue().reversed()) + .forEach( + entry -> + relationRows.add(Arrays.asList(entry.getKey(), entry.getValue().toString()))); + + printToAsciiTable(relationColumns, relationRows, "No relation type statistics"); + } + } + + private String getRelationshipName(int relation) { + // Map common relationship types to names + // These constants should ideally be imported from the actual schema + return switch (relation) { + case 10 -> "CONTAINS"; + case 11 -> "CREATED_BY"; + case 12 -> "MENTIONED_IN"; + case 13 -> "PARENT_OF"; + case 14 -> "OWNS"; + case 15 -> "FOLLOWS"; + case 16 -> "JOINED"; + case 17 -> "REACTED_TO"; + case 18 -> "REPLIED_TO"; + case 19 -> "TESTED_BY"; + case 20 -> "UPSTREAM"; + case 21 -> "DOWNSTREAM"; + default -> "RELATION_" + relation; + }; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index fce031df4b0..6a740e4802a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -140,7 +140,8 @@ public class OpenMetadataOperations implements Callable { public Integer call() { LOG.info( "Subcommand needed: 'info', 'validate', 'repair', 'check-connection', " - + "'drop-create', 'changelog', 'migrate', 'migrate-secrets', 'reindex', 'deploy-pipelines'"); + + "'drop-create', 'changelog', 'migrate', 'migrate-secrets', 'reindex', 'deploy-pipelines', " + + "'dbServiceCleanup', 'relationshipCleanup'"); return 0; } @@ -709,6 +710,48 @@ public class OpenMetadataOperations implements Callable { } } + @Command( + name = "relationshipCleanup", + description = + "Cleans up orphaned entity relationships where referenced entities no longer exist. By default, runs in dry-run mode to only identify orphaned relationships.") + public Integer cleanupOrphanedRelationships( + @Option( + names = {"--delete"}, + description = + "Actually delete the orphaned relationships. Without this flag, the command only identifies orphaned relationships (dry-run mode).", + defaultValue = "false") + boolean delete, + @Option( + names = {"-b", "--batch-size"}, + defaultValue = "1000", + description = "Number of relationships to process in each batch.") + int batchSize) { + try { + boolean dryRun = !delete; + LOG.info( + "Running Entity Relationship Cleanup. Dry run: {}, Batch size: {}", dryRun, batchSize); + parseConfig(); + + EntityRelationshipCleanup cleanup = new EntityRelationshipCleanup(collectionDAO, dryRun); + EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(batchSize); + + LOG.info("=== Entity Relationship Cleanup Summary ==="); + LOG.info("Total relationships scanned: {}", result.getTotalRelationshipsScanned()); + LOG.info("Orphaned relationships found: {}", result.getOrphanedRelationshipsFound()); + LOG.info("Relationships deleted: {}", result.getRelationshipsDeleted()); + + if (dryRun && result.getOrphanedRelationshipsFound() > 0) { + LOG.info("To actually delete these orphaned relationships, run with --delete"); + return 1; + } + + return 0; + } catch (Exception e) { + LOG.error("Failed to cleanup orphaned relationships due to ", e); + return 1; + } + } + @Command(name = "reindex", description = "Re Indexes data into search engine from command line.") public Integer reIndex( @Option( diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/EntityRelationshipCleanupTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/EntityRelationshipCleanupTest.java new file mode 100644 index 00000000000..d2670101dc2 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/EntityRelationshipCleanupTest.java @@ -0,0 +1,473 @@ +/* + * Copyright 2025 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.openmetadata.schema.api.data.CreateTable; +import org.openmetadata.schema.entity.data.Database; +import org.openmetadata.schema.entity.data.DatabaseSchema; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.service.Entity; +import org.openmetadata.service.OpenMetadataApplicationTest; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.resources.EntityResourceTest; +import org.openmetadata.service.resources.databases.TableResourceTest; +import org.openmetadata.service.resources.services.DatabaseServiceResourceTest; + +@Slf4j +@Execution(ExecutionMode.CONCURRENT) +class EntityRelationshipCleanupTest extends OpenMetadataApplicationTest { + + private static TableResourceTest tableTest; + + private static List testTables; + + private static CollectionDAO collectionDAO; + private static EntityRelationshipCleanup cleanup; + + @BeforeAll + static void setup(TestInfo test) throws IOException { + DatabaseServiceResourceTest serviceTest = new DatabaseServiceResourceTest(); + tableTest = new TableResourceTest(); + testTables = new ArrayList<>(); + + collectionDAO = Entity.getCollectionDAO(); + serviceTest.setupDatabaseServices(test); + tableTest.setupDatabaseSchemas(test); + setupTestEntities(test); + } + + private static void setupTestEntities(TestInfo test) throws IOException { + Database testDatabase = EntityResourceTest.DATABASE; + DatabaseSchema testSchema = EntityResourceTest.DATABASE_SCHEMA; + + for (int i = 0; i < 3; i++) { + CreateTable createTable = + tableTest.createRequest(test, i).withDatabaseSchema(testSchema.getFullyQualifiedName()); + Table table = tableTest.createEntity(createTable, ADMIN_AUTH_HEADERS); + testTables.add(table); + } + + LOG.info( + "Created test entities: Database={}, Schema={}, Tables={}", + testDatabase.getId(), + testSchema.getId(), + testTables.size()); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_cleanupWithValidRelationships_shouldFindNoOrphans() { + cleanup = new EntityRelationshipCleanup(collectionDAO, true); // dry-run mode + EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(100); + assertNotNull(result); + assertTrue(result.getTotalRelationshipsScanned() >= 0); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_cleanupAfterDeletingEntity_shouldDetectOrphans() { + Table tableToDelete = testTables.get(0); + UUID tableId = tableToDelete.getId(); + collectionDAO.tableDAO().delete(tableId); + cleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(100); + assertNotNull(result); + assertTrue( + result.getOrphanedRelationshipsFound() > 0, + "Should find orphaned relationships after entity deletion"); + boolean foundOrphanedTable = + result.getOrphanedRelationships().stream() + .anyMatch( + orphan -> + tableId.toString().equals(orphan.getFromId()) + || tableId.toString().equals(orphan.getToId())); + assertTrue(foundOrphanedTable, "Should find orphaned relationships for deleted table"); + + assertFalse(result.getOrphansByEntityType().isEmpty(), "Should have entity type statistics"); + assertFalse( + result.getOrphansByRelationType().isEmpty(), "Should have relation type statistics"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_actualCleanup_shouldDeleteOrphanedRelationships() { + EntityRelationshipCleanup dryRunCleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult dryRunResult = dryRunCleanup.performCleanup(100); + + int orphansFoundInDryRun = dryRunResult.getOrphanedRelationshipsFound(); + + if (orphansFoundInDryRun > 0) { + EntityRelationshipCleanup actualCleanup = new EntityRelationshipCleanup(collectionDAO, false); + EntityRelationshipCleanup.CleanupResult actualResult = actualCleanup.performCleanup(100); + + assertNotNull(actualResult); + assertTrue( + actualResult.getRelationshipsDeleted() > 0, "Should have deleted orphaned relationships"); + assertEquals( + orphansFoundInDryRun, + actualResult.getRelationshipsDeleted(), + "Should delete same number of relationships as found in dry run"); + + EntityRelationshipCleanup verificationCleanup = + new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult verificationResult = + verificationCleanup.performCleanup(100); + + assertEquals( + 0, + verificationResult.getOrphanedRelationshipsFound(), + "Should find no orphaned relationships after cleanup"); + } + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_paginationWithLargeBatchSize() { + cleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(10000); + assertNotNull(result); + assertTrue(result.getTotalRelationshipsScanned() >= 0); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_paginationWithSmallBatchSize() { + cleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(10); + assertNotNull(result); + assertTrue(result.getTotalRelationshipsScanned() >= 0); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_createOrphanedRelationshipScenario() { + UUID nonExistentId1 = UUID.randomUUID(); + UUID nonExistentId2 = UUID.randomUUID(); + + collectionDAO + .relationshipDAO() + .insert( + nonExistentId1, + nonExistentId2, + "table", + "databaseSchema", + Relationship.CONTAINS.ordinal(), + null); + cleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(100); + assertTrue( + result.getOrphanedRelationshipsFound() > 0, + "Should detect manually created orphaned relationship"); + + boolean foundSpecificOrphan = + result.getOrphanedRelationships().stream() + .anyMatch( + orphan -> + nonExistentId1.toString().equals(orphan.getFromId()) + && nonExistentId2.toString().equals(orphan.getToId())); + + assertTrue(foundSpecificOrphan, "Should find the specific orphaned relationship created"); + + EntityRelationshipCleanup actualCleanup = new EntityRelationshipCleanup(collectionDAO, false); + EntityRelationshipCleanup.CleanupResult cleanupResult = actualCleanup.performCleanup(100); + + assertTrue( + cleanupResult.getRelationshipsDeleted() > 0, "Should delete the orphaned relationship"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_validationOfExistingRelationships() { + long relationshipCountBefore = collectionDAO.relationshipDAO().getTotalRelationshipCount(); + cleanup = new EntityRelationshipCleanup(collectionDAO, false); + EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(100); + long relationshipCountAfter = collectionDAO.relationshipDAO().getTotalRelationshipCount(); + long expectedCount = relationshipCountBefore - result.getRelationshipsDeleted(); + assertEquals( + expectedCount, + relationshipCountAfter, + "Relationship count should match expected after cleanup"); + + LOG.info( + "Validation test - Before: {}, After: {}, Deleted: {}", + relationshipCountBefore, + relationshipCountAfter, + result.getRelationshipsDeleted()); + + for (Table table : testTables.subList(1, testTables.size())) { + try { + Table retrievedTable = tableTest.getEntity(table.getId(), ADMIN_AUTH_HEADERS); + assertNotNull(retrievedTable, "Valid table should still exist"); + assertNotNull( + retrievedTable.getDatabaseSchema(), "Table should still have schema reference"); + } catch (Exception e) { + // If table was part of orphaned cleanup, that's expected + LOG.info("Table {} not found after cleanup (expected if it was orphaned)", table.getId()); + } + } + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_relationshipCleanupCommand_dryRun() { + EntityRelationshipCleanup dryRunCleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult result = dryRunCleanup.performCleanup(500); + assertNotNull(result, "Cleanup result should not be null"); + assertTrue(result.getTotalRelationshipsScanned() >= 0, "Should scan some relationships"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_relationshipCleanupCommand_withOrphanedData() { + UUID nonExistentEntityId = UUID.randomUUID(); + + collectionDAO + .relationshipDAO() + .insert( + testTables.get(1).getId(), + nonExistentEntityId, + "table", + "databaseSchema", + Relationship.CONTAINS.ordinal(), + null); + + EntityRelationshipCleanup dryRunCleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult dryRunResult = dryRunCleanup.performCleanup(100); + + assertNotNull(dryRunResult, "Dry-run result should not be null"); + assertTrue(dryRunResult.getOrphanedRelationshipsFound() > 0, "Should find orphaned data"); + + EntityRelationshipCleanup actualCleanup = new EntityRelationshipCleanup(collectionDAO, false); + EntityRelationshipCleanup.CleanupResult cleanupResult = actualCleanup.performCleanup(100); + + assertNotNull(cleanupResult, "Cleanup result should not be null"); + assertTrue(cleanupResult.getRelationshipsDeleted() > 0, "Should delete orphaned relationships"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_relationshipCleanupCommand_noOrphanedData() { + EntityRelationshipCleanup cleanup1 = new EntityRelationshipCleanup(collectionDAO, false); + EntityRelationshipCleanup.CleanupResult result = cleanup1.performCleanup(100); + assertNotNull(result, "Cleanup result should not be null"); + assertEquals( + 0, + result.getRelationshipsDeleted(), + "Should not delete any relationships when none are orphaned"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_relationshipCleanupCommand_smallBatchSize() { + EntityRelationshipCleanup cleanup1 = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult result = cleanup1.performCleanup(10); + + assertNotNull(result, "Cleanup result should not be null"); + assertTrue(result.getTotalRelationshipsScanned() >= 0, "Should scan relationships"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_relationshipCleanupCommand_largeBatchSize() { + EntityRelationshipCleanup cleanup1 = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult result = cleanup1.performCleanup(10000); + + assertNotNull(result, "Cleanup result should not be null"); + assertTrue(result.getTotalRelationshipsScanned() >= 0, "Should scan relationships"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_relationshipCleanupCommand_multipleOrphanedRelationships() { + for (int i = 0; i < 5; i++) { + UUID nonExistentId1 = UUID.randomUUID(); + UUID nonExistentId2 = UUID.randomUUID(); + + collectionDAO + .relationshipDAO() + .insert( + nonExistentId1, + nonExistentId2, + "table", + "databaseSchema", + Relationship.CONTAINS.ordinal(), + null); + } + + EntityRelationshipCleanup cleanup1 = new EntityRelationshipCleanup(collectionDAO, false); + EntityRelationshipCleanup.CleanupResult result = cleanup1.performCleanup(2); + + assertNotNull(result, "Cleanup result should not be null"); + assertTrue( + result.getRelationshipsDeleted() >= 5, + "Should delete at least the 5 orphaned relationships created"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_relationshipCleanupCommand_validationOfParameters() { + + EntityRelationshipCleanup minBatchCleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult minBatchResult = minBatchCleanup.performCleanup(1); + + assertNotNull(minBatchResult, "Minimum batch size result should not be null"); + EntityRelationshipCleanup defaultCleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult defaultResult = defaultCleanup.performCleanup(1000); + + assertNotNull(defaultResult, "Default batch size result should not be null"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_commandIntegrationValidation() { + EntityRelationshipCleanup cleanup1 = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult result = cleanup1.performCleanup(100); + + assertNotNull(result, "Command integration result should not be null"); + assertTrue(result.getTotalRelationshipsScanned() >= 0, "Should scan relationships"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_emptyDatabaseScenario() { + // Test cleanup behavior when there are minimal relationships + + // Run cleanup on current database + cleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(100); + + assertNotNull(result); + assertTrue(result.getTotalRelationshipsScanned() >= 0); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_commandBehavior_defaultDryRunWithNoOrphans() { + EntityRelationshipCleanup dryRunCleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult result = dryRunCleanup.performCleanup(100); + + assertNotNull(result); + + boolean wouldReturnOne = result.getOrphanedRelationshipsFound() > 0; + assertFalse( + wouldReturnOne || result.getOrphanedRelationshipsFound() < 0, + "Default dry-run with no orphans should indicate success (exit code 0)"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_commandBehavior_defaultDryRunWithOrphans() { + UUID nonExistentId1 = UUID.randomUUID(); + UUID nonExistentId2 = UUID.randomUUID(); + + collectionDAO + .relationshipDAO() + .insert( + nonExistentId1, + nonExistentId2, + "table", + "databaseSchema", + Relationship.CONTAINS.ordinal(), + null); + + EntityRelationshipCleanup dryRunCleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult result = dryRunCleanup.performCleanup(100); + + assertNotNull(result); + + boolean wouldReturnOne = result.getOrphanedRelationshipsFound() > 0; + assertTrue( + wouldReturnOne, "Default dry-run with orphans found should indicate failure (exit code 1)"); + + assertTrue(result.getOrphanedRelationshipsFound() > 0, "Should find the orphaned relationship"); + assertEquals(0, result.getRelationshipsDeleted(), "Should not delete in dry-run mode"); + + EntityRelationshipCleanup actualCleanup = new EntityRelationshipCleanup(collectionDAO, false); + actualCleanup.performCleanup(100); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_commandBehavior_explicitDeleteFlag() { + UUID nonExistentId1 = UUID.randomUUID(); + UUID nonExistentId2 = UUID.randomUUID(); + + collectionDAO + .relationshipDAO() + .insert( + nonExistentId1, + nonExistentId2, + "table", + "databaseSchema", + Relationship.CONTAINS.ordinal(), + null); + + EntityRelationshipCleanup deleteCleanup = new EntityRelationshipCleanup(collectionDAO, false); + EntityRelationshipCleanup.CleanupResult result = deleteCleanup.performCleanup(100); + + assertNotNull(result); + assertTrue(result.getRelationshipsDeleted() > 0, "Should have deleted orphaned relationships"); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_commandBehavior_batchSizeParameter() { + int customBatchSize = 50; + + EntityRelationshipCleanup cleanup = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult result = cleanup.performCleanup(customBatchSize); + + assertNotNull(result); + assertTrue(result.getTotalRelationshipsScanned() >= 0); + } + + @Test + @Execution(ExecutionMode.CONCURRENT) + void test_commandBehavior_flagSemantics() { + EntityRelationshipCleanup defaultBehavior = new EntityRelationshipCleanup(collectionDAO, true); + EntityRelationshipCleanup.CleanupResult defaultResult = defaultBehavior.performCleanup(100); + + EntityRelationshipCleanup deleteBehavior = new EntityRelationshipCleanup(collectionDAO, false); + EntityRelationshipCleanup.CleanupResult deleteResult = deleteBehavior.performCleanup(100); + + assertNotNull(defaultResult); + assertNotNull(deleteResult); + + assertEquals( + 0, + defaultResult.getRelationshipsDeleted(), + "Default behavior (dry-run) should not delete any relationships"); + + assertTrue( + deleteResult.getRelationshipsDeleted() >= 0, + "Delete mode should delete 0 or more relationships"); + } +}