diff --git a/bootstrap/sql/migrations/native/1.11.2/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.11.2/mysql/schemaChanges.sql new file mode 100644 index 00000000000..3d167dfd14d --- /dev/null +++ b/bootstrap/sql/migrations/native/1.11.2/mysql/schemaChanges.sql @@ -0,0 +1,3 @@ +-- No schema changes required for this migration +-- This migration only fixes FQN hash for entities with service names containing dots +-- The data migration is handled in Java code diff --git a/bootstrap/sql/migrations/native/1.11.2/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.11.2/postgres/schemaChanges.sql new file mode 100644 index 00000000000..3d167dfd14d --- /dev/null +++ b/bootstrap/sql/migrations/native/1.11.2/postgres/schemaChanges.sql @@ -0,0 +1,3 @@ +-- No schema changes required for this migration +-- This migration only fixes FQN hash for entities with service names containing dots +-- The data migration is handled in Java code diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APICollectionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APICollectionRepository.java index 1c508d2fe52..6d115492125 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APICollectionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APICollectionRepository.java @@ -48,7 +48,8 @@ public class APICollectionRepository extends EntityRepository { @Override public void setFullyQualifiedName(APICollection apiCollection) { apiCollection.setFullyQualifiedName( - FullyQualifiedName.build(apiCollection.getService().getName(), apiCollection.getName())); + FullyQualifiedName.add( + apiCollection.getService().getFullyQualifiedName(), apiCollection.getName())); } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java index d0387d0faf0..59be215d069 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DashboardDataModelRepository.java @@ -68,9 +68,11 @@ public class DashboardDataModelRepository extends EntityRepository { @Override public void setFullyQualifiedName(Database database) { database.setFullyQualifiedName( - FullyQualifiedName.build(database.getService().getName(), database.getName())); + FullyQualifiedName.add(database.getService().getFullyQualifiedName(), database.getName())); } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1112/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1112/Migration.java new file mode 100644 index 00000000000..df5139b1a44 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1112/Migration.java @@ -0,0 +1,40 @@ +package org.openmetadata.service.migration.mysql.v1112; + +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixApiCollectionFqnHash; +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixApiEndpointFqnHash; +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixDashboardDataModelFqnHash; +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixDatabaseFqnHash; +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixDatabaseSchemaFqnHash; +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixStoredProcedureFqnHash; +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixTableFqnHash; + +import lombok.SneakyThrows; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; + +public class Migration extends MigrationProcessImpl { + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + @SneakyThrows + public void runDataMigration() { + // Fix FQN and hash for entities with services that have dots in their names. + // Order matters - parent entities must be fixed before child entities. + + // Database service hierarchy: Service -> Database -> Schema -> Table/StoredProcedure + fixDatabaseFqnHash(handle, collectionDAO); + fixDatabaseSchemaFqnHash(handle, collectionDAO); + fixTableFqnHash(handle, collectionDAO); + fixStoredProcedureFqnHash(handle, collectionDAO); + + // Dashboard service hierarchy: Service -> DashboardDataModel + fixDashboardDataModelFqnHash(handle, collectionDAO); + + // API service hierarchy: Service -> APICollection -> APIEndpoint + fixApiCollectionFqnHash(handle, collectionDAO); + fixApiEndpointFqnHash(handle, collectionDAO); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1120/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1120/Migration.java new file mode 100644 index 00000000000..742448c4284 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1120/Migration.java @@ -0,0 +1,40 @@ +package org.openmetadata.service.migration.mysql.v1120; + +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixApiCollectionFqnHash; +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixApiEndpointFqnHash; +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixDashboardDataModelFqnHash; +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixDatabaseFqnHash; +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixDatabaseSchemaFqnHash; +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixStoredProcedureFqnHash; +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixTableFqnHash; + +import lombok.SneakyThrows; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; + +public class Migration extends MigrationProcessImpl { + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + @SneakyThrows + public void runDataMigration() { + // Fix FQN and hash for entities with services that have dots in their names. + // Order matters - parent entities must be fixed before child entities. + + // Database service hierarchy: Service -> Database -> Schema -> Table/StoredProcedure + fixDatabaseFqnHash(handle, collectionDAO); + fixDatabaseSchemaFqnHash(handle, collectionDAO); + fixTableFqnHash(handle, collectionDAO); + fixStoredProcedureFqnHash(handle, collectionDAO); + + // Dashboard service hierarchy: Service -> DashboardDataModel + fixDashboardDataModelFqnHash(handle, collectionDAO); + + // API service hierarchy: Service -> APICollection -> APIEndpoint + fixApiCollectionFqnHash(handle, collectionDAO); + fixApiEndpointFqnHash(handle, collectionDAO); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1112/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1112/Migration.java new file mode 100644 index 00000000000..04d27b0b178 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1112/Migration.java @@ -0,0 +1,40 @@ +package org.openmetadata.service.migration.postgres.v1112; + +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixApiCollectionFqnHash; +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixApiEndpointFqnHash; +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixDashboardDataModelFqnHash; +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixDatabaseFqnHash; +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixDatabaseSchemaFqnHash; +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixStoredProcedureFqnHash; +import static org.openmetadata.service.migration.utils.v1112.MigrationUtil.fixTableFqnHash; + +import lombok.SneakyThrows; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; + +public class Migration extends MigrationProcessImpl { + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + @SneakyThrows + public void runDataMigration() { + // Fix FQN and hash for entities with services that have dots in their names. + // Order matters - parent entities must be fixed before child entities. + + // Database service hierarchy: Service -> Database -> Schema -> Table/StoredProcedure + fixDatabaseFqnHash(handle, collectionDAO); + fixDatabaseSchemaFqnHash(handle, collectionDAO); + fixTableFqnHash(handle, collectionDAO); + fixStoredProcedureFqnHash(handle, collectionDAO); + + // Dashboard service hierarchy: Service -> DashboardDataModel + fixDashboardDataModelFqnHash(handle, collectionDAO); + + // API service hierarchy: Service -> APICollection -> APIEndpoint + fixApiCollectionFqnHash(handle, collectionDAO); + fixApiEndpointFqnHash(handle, collectionDAO); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1120/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1120/Migration.java new file mode 100644 index 00000000000..51c962957af --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1120/Migration.java @@ -0,0 +1,40 @@ +package org.openmetadata.service.migration.postgres.v1120; + +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixApiCollectionFqnHash; +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixApiEndpointFqnHash; +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixDashboardDataModelFqnHash; +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixDatabaseFqnHash; +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixDatabaseSchemaFqnHash; +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixStoredProcedureFqnHash; +import static org.openmetadata.service.migration.utils.v1120.MigrationUtil.fixTableFqnHash; + +import lombok.SneakyThrows; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; + +public class Migration extends MigrationProcessImpl { + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + @SneakyThrows + public void runDataMigration() { + // Fix FQN and hash for entities with services that have dots in their names. + // Order matters - parent entities must be fixed before child entities. + + // Database service hierarchy: Service -> Database -> Schema -> Table/StoredProcedure + fixDatabaseFqnHash(handle, collectionDAO); + fixDatabaseSchemaFqnHash(handle, collectionDAO); + fixTableFqnHash(handle, collectionDAO); + fixStoredProcedureFqnHash(handle, collectionDAO); + + // Dashboard service hierarchy: Service -> DashboardDataModel + fixDashboardDataModelFqnHash(handle, collectionDAO); + + // API service hierarchy: Service -> APICollection -> APIEndpoint + fixApiCollectionFqnHash(handle, collectionDAO); + fixApiEndpointFqnHash(handle, collectionDAO); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1112/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1112/MigrationUtil.java new file mode 100644 index 00000000000..f39d27a2af1 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1112/MigrationUtil.java @@ -0,0 +1,505 @@ +package org.openmetadata.service.migration.utils.v1112; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.core.Handle; +import org.openmetadata.schema.entity.data.APICollection; +import org.openmetadata.schema.entity.data.APIEndpoint; +import org.openmetadata.schema.entity.data.DashboardDataModel; +import org.openmetadata.schema.entity.data.Database; +import org.openmetadata.schema.entity.data.DatabaseSchema; +import org.openmetadata.schema.entity.data.StoredProcedure; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.ColumnUtil; +import org.openmetadata.service.util.FullyQualifiedName; + +/** + * Migration utility to fix FQN and FQN hash for entities whose parent service name contains dots. + * + *

Prior to this fix, the FQN was incorrectly built using getService().getName() instead of + * getService().getFullyQualifiedName(). This caused service names with dots (e.g., "my.service") + * to be incorrectly represented in child entity FQNs. + * + *

For example, if service FQN is "my.service" (quoted as "\"my.service\""), a Database named + * "mydb" would incorrectly have FQN "my.service.mydb" instead of the correct "\"my.service\".mydb" + */ +@Slf4j +public class MigrationUtil { + + private MigrationUtil() {} + + /** + * Find all service IDs that have dots in their names. These are the services whose child entities + * need FQN fixes. + */ + private static Set findServicesWithDotsInName(Handle handle, String serviceTable) { + Set serviceIds = new HashSet<>(); + // Query for services where the name contains a dot (using the virtual 'name' column) + String query = String.format("SELECT id FROM %s WHERE name LIKE '%%.%%'", serviceTable); + + try { + List> rows = handle.createQuery(query).mapToMap().list(); + for (Map row : rows) { + Object idObj = row.get("id"); + if (idObj != null) { + serviceIds.add(UUID.fromString(idObj.toString())); + } + } + } catch (Exception e) { + LOG.warn( + "Error finding services with dots in name from {}: {}", serviceTable, e.getMessage()); + } + + return serviceIds; + } + + /** + * Find entity IDs that belong to the given parent entities via relationship table. + */ + private static Set findChildEntityIds( + CollectionDAO collectionDAO, Set parentIds, String fromType, String toType) { + Set childIds = new HashSet<>(); + for (UUID parentId : parentIds) { + List records = + collectionDAO + .relationshipDAO() + .findTo(parentId, fromType, Relationship.CONTAINS.ordinal(), toType); + for (CollectionDAO.EntityRelationshipRecord record : records) { + childIds.add(record.getId()); + } + } + return childIds; + } + + /** Fix FQN and hash for Database entities that have services with dots in their names. */ + public static void fixDatabaseFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info("Starting migration to fix Database FQN hash for services with dots in names"); + + // Find all database services with dots in their names + Set serviceIds = findServicesWithDotsInName(handle, "dbservice_entity"); + if (serviceIds.isEmpty()) { + LOG.info("No database services with dots in names found. Skipping Database FQN fix."); + return; + } + LOG.info("Found {} database services with dots in names", serviceIds.size()); + + int fixedCount = 0; + // Process each service and its databases + for (UUID serviceId : serviceIds) { + try { + // Get the service entity to get its FQN + var service = collectionDAO.dbServiceDAO().findEntityById(serviceId); + if (service == null) continue; + + String serviceFqn = service.getFullyQualifiedName(); + if (serviceFqn == null || !serviceFqn.contains("\"")) continue; + + // Find all databases belonging to this service + Set databaseIds = + findChildEntityIds( + collectionDAO, Set.of(serviceId), Entity.DATABASE_SERVICE, Entity.DATABASE); + + for (UUID databaseId : databaseIds) { + try { + Database database = collectionDAO.databaseDAO().findEntityById(databaseId); + if (database == null) continue; + + String expectedFqn = FullyQualifiedName.add(serviceFqn, database.getName()); + String currentFqn = database.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing Database FQN: {} -> {}", currentFqn, expectedFqn); + database.setFullyQualifiedName(expectedFqn); + collectionDAO.databaseDAO().update(database); + fixedCount++; + } + } catch (Exception e) { + LOG.warn("Error processing database entity {}: {}", databaseId, e.getMessage()); + } + } + } catch (Exception e) { + LOG.warn("Error processing service {}: {}", serviceId, e.getMessage()); + } + } + + LOG.info("Fixed {} Database entities with incorrect FQN hash", fixedCount); + } + + /** + * Fix FQN and hash for DatabaseSchema entities whose parent database has a service with dots in + * its name. Must be called after fixDatabaseFqnHash. + */ + public static void fixDatabaseSchemaFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info("Starting migration to fix DatabaseSchema FQN hash for services with dots in names"); + + // Find all database services with dots in their names + Set serviceIds = findServicesWithDotsInName(handle, "dbservice_entity"); + if (serviceIds.isEmpty()) { + LOG.info("No database services with dots in names found. Skipping DatabaseSchema FQN fix."); + return; + } + + // Find all databases belonging to these services + Set databaseIds = + findChildEntityIds(collectionDAO, serviceIds, Entity.DATABASE_SERVICE, Entity.DATABASE); + if (databaseIds.isEmpty()) { + LOG.info("No databases found under services with dots. Skipping DatabaseSchema FQN fix."); + return; + } + + // Find all schemas belonging to these databases + Set schemaIds = + findChildEntityIds(collectionDAO, databaseIds, Entity.DATABASE, Entity.DATABASE_SCHEMA); + LOG.info("Found {} database schemas to check", schemaIds.size()); + + int fixedCount = 0; + for (UUID schemaId : schemaIds) { + try { + DatabaseSchema schema = collectionDAO.databaseSchemaDAO().findEntityById(schemaId); + if (schema == null) continue; + + // Get the database reference and its FQN + Database database = + collectionDAO.databaseDAO().findEntityById(schema.getDatabase().getId()); + if (database == null) continue; + + String databaseFqn = database.getFullyQualifiedName(); + if (databaseFqn == null || !databaseFqn.contains("\"")) continue; + + String expectedFqn = FullyQualifiedName.add(databaseFqn, schema.getName()); + String currentFqn = schema.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing DatabaseSchema FQN: {} -> {}", currentFqn, expectedFqn); + schema.setFullyQualifiedName(expectedFqn); + collectionDAO.databaseSchemaDAO().update(schema); + fixedCount++; + } + } catch (Exception e) { + LOG.warn("Error processing DatabaseSchema entity {}: {}", schemaId, e.getMessage()); + } + } + + LOG.info("Fixed {} DatabaseSchema entities with incorrect FQN hash", fixedCount); + } + + /** + * Fix FQN and hash for Table entities whose parent schema has a service with dots in its name. + * Must be called after fixDatabaseSchemaFqnHash. + */ + public static void fixTableFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info("Starting migration to fix Table FQN hash for services with dots in names"); + + // Find all database services with dots in their names + Set serviceIds = findServicesWithDotsInName(handle, "dbservice_entity"); + if (serviceIds.isEmpty()) { + LOG.info("No database services with dots in names found. Skipping Table FQN fix."); + return; + } + + // Find all databases belonging to these services + Set databaseIds = + findChildEntityIds(collectionDAO, serviceIds, Entity.DATABASE_SERVICE, Entity.DATABASE); + if (databaseIds.isEmpty()) { + LOG.info("No databases found under services with dots. Skipping Table FQN fix."); + return; + } + + // Find all schemas belonging to these databases + Set schemaIds = + findChildEntityIds(collectionDAO, databaseIds, Entity.DATABASE, Entity.DATABASE_SCHEMA); + if (schemaIds.isEmpty()) { + LOG.info("No schemas found under services with dots. Skipping Table FQN fix."); + return; + } + + // Find all tables belonging to these schemas + Set tableIds = + findChildEntityIds(collectionDAO, schemaIds, Entity.DATABASE_SCHEMA, Entity.TABLE); + LOG.info("Found {} tables to check", tableIds.size()); + + int fixedCount = 0; + for (UUID tableId : tableIds) { + try { + Table table = collectionDAO.tableDAO().findEntityById(tableId); + if (table == null) continue; + + // Get the schema FQN + DatabaseSchema schema = + collectionDAO.databaseSchemaDAO().findEntityById(table.getDatabaseSchema().getId()); + if (schema == null) continue; + + String schemaFqn = schema.getFullyQualifiedName(); + if (schemaFqn == null || !schemaFqn.contains("\"")) continue; + + String expectedFqn = FullyQualifiedName.add(schemaFqn, table.getName()); + String currentFqn = table.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing Table FQN: {} -> {}", currentFqn, expectedFqn); + table.setFullyQualifiedName(expectedFqn); + // Also fix column FQNs + ColumnUtil.setColumnFQN(expectedFqn, table.getColumns()); + collectionDAO.tableDAO().update(table); + fixedCount++; + } + } catch (Exception e) { + LOG.warn("Error processing Table entity {}: {}", tableId, e.getMessage()); + } + } + + LOG.info("Fixed {} Table entities with incorrect FQN hash", fixedCount); + } + + /** + * Fix FQN and hash for StoredProcedure entities whose parent schema has a service with dots in + * its name. Must be called after fixDatabaseSchemaFqnHash. + */ + public static void fixStoredProcedureFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info("Starting migration to fix StoredProcedure FQN hash for services with dots in names"); + + // Find all database services with dots in their names + Set serviceIds = findServicesWithDotsInName(handle, "dbservice_entity"); + if (serviceIds.isEmpty()) { + LOG.info("No database services with dots in names found. Skipping StoredProcedure FQN fix."); + return; + } + + // Find all databases belonging to these services + Set databaseIds = + findChildEntityIds(collectionDAO, serviceIds, Entity.DATABASE_SERVICE, Entity.DATABASE); + if (databaseIds.isEmpty()) { + LOG.info("No databases found under services with dots. Skipping StoredProcedure FQN fix."); + return; + } + + // Find all schemas belonging to these databases + Set schemaIds = + findChildEntityIds(collectionDAO, databaseIds, Entity.DATABASE, Entity.DATABASE_SCHEMA); + if (schemaIds.isEmpty()) { + LOG.info("No schemas found under services with dots. Skipping StoredProcedure FQN fix."); + return; + } + + // Find all stored procedures belonging to these schemas + Set spIds = + findChildEntityIds( + collectionDAO, schemaIds, Entity.DATABASE_SCHEMA, Entity.STORED_PROCEDURE); + LOG.info("Found {} stored procedures to check", spIds.size()); + + int fixedCount = 0; + for (UUID spId : spIds) { + try { + StoredProcedure sp = collectionDAO.storedProcedureDAO().findEntityById(spId); + if (sp == null) continue; + + // Get the schema FQN + DatabaseSchema schema = + collectionDAO.databaseSchemaDAO().findEntityById(sp.getDatabaseSchema().getId()); + if (schema == null) continue; + + String schemaFqn = schema.getFullyQualifiedName(); + if (schemaFqn == null || !schemaFqn.contains("\"")) continue; + + String expectedFqn = FullyQualifiedName.add(schemaFqn, sp.getName()); + String currentFqn = sp.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing StoredProcedure FQN: {} -> {}", currentFqn, expectedFqn); + sp.setFullyQualifiedName(expectedFqn); + collectionDAO.storedProcedureDAO().update(sp); + fixedCount++; + } + } catch (Exception e) { + LOG.warn("Error processing StoredProcedure entity {}: {}", spId, e.getMessage()); + } + } + + LOG.info("Fixed {} StoredProcedure entities with incorrect FQN hash", fixedCount); + } + + /** + * Fix FQN and hash for DashboardDataModel entities that have services with dots in their names. + */ + public static void fixDashboardDataModelFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info( + "Starting migration to fix DashboardDataModel FQN hash for services with dots in names"); + + // Find all dashboard services with dots in their names + Set serviceIds = findServicesWithDotsInName(handle, "dashboard_service_entity"); + if (serviceIds.isEmpty()) { + LOG.info( + "No dashboard services with dots in names found. Skipping DashboardDataModel FQN fix."); + return; + } + LOG.info("Found {} dashboard services with dots in names", serviceIds.size()); + + int fixedCount = 0; + // Process each service and its data models + for (UUID serviceId : serviceIds) { + try { + // Get the service entity to get its FQN + var service = collectionDAO.dashboardServiceDAO().findEntityById(serviceId); + if (service == null) continue; + + String serviceFqn = service.getFullyQualifiedName(); + if (serviceFqn == null || !serviceFqn.contains("\"")) continue; + + // Find all data models belonging to this service + Set dataModelIds = + findChildEntityIds( + collectionDAO, + Set.of(serviceId), + Entity.DASHBOARD_SERVICE, + Entity.DASHBOARD_DATA_MODEL); + + for (UUID dataModelId : dataModelIds) { + try { + DashboardDataModel dataModel = + collectionDAO.dashboardDataModelDAO().findEntityById(dataModelId); + if (dataModel == null) continue; + + String expectedFqn = FullyQualifiedName.add(serviceFqn + ".model", dataModel.getName()); + String currentFqn = dataModel.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing DashboardDataModel FQN: {} -> {}", currentFqn, expectedFqn); + dataModel.setFullyQualifiedName(expectedFqn); + ColumnUtil.setColumnFQN(expectedFqn, dataModel.getColumns()); + collectionDAO.dashboardDataModelDAO().update(dataModel); + fixedCount++; + } + } catch (Exception e) { + LOG.warn( + "Error processing DashboardDataModel entity {}: {}", dataModelId, e.getMessage()); + } + } + } catch (Exception e) { + LOG.warn("Error processing service {}: {}", serviceId, e.getMessage()); + } + } + + LOG.info("Fixed {} DashboardDataModel entities with incorrect FQN hash", fixedCount); + } + + /** Fix FQN and hash for APICollection entities that have services with dots in their names. */ + public static void fixApiCollectionFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info("Starting migration to fix APICollection FQN hash for services with dots in names"); + + // Find all API services with dots in their names + Set serviceIds = findServicesWithDotsInName(handle, "api_service_entity"); + if (serviceIds.isEmpty()) { + LOG.info("No API services with dots in names found. Skipping APICollection FQN fix."); + return; + } + LOG.info("Found {} API services with dots in names", serviceIds.size()); + + int fixedCount = 0; + // Process each service and its collections + for (UUID serviceId : serviceIds) { + try { + // Get the service entity to get its FQN + var service = collectionDAO.apiServiceDAO().findEntityById(serviceId); + if (service == null) continue; + + String serviceFqn = service.getFullyQualifiedName(); + if (serviceFqn == null || !serviceFqn.contains("\"")) continue; + + // Find all API collections belonging to this service + Set collectionIds = + findChildEntityIds( + collectionDAO, Set.of(serviceId), Entity.API_SERVICE, Entity.API_COLLECTION); + + for (UUID collectionId : collectionIds) { + try { + APICollection apiCollection = + collectionDAO.apiCollectionDAO().findEntityById(collectionId); + if (apiCollection == null) continue; + + String expectedFqn = FullyQualifiedName.add(serviceFqn, apiCollection.getName()); + String currentFqn = apiCollection.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing APICollection FQN: {} -> {}", currentFqn, expectedFqn); + apiCollection.setFullyQualifiedName(expectedFqn); + collectionDAO.apiCollectionDAO().update(apiCollection); + fixedCount++; + } + } catch (Exception e) { + LOG.warn("Error processing APICollection entity {}: {}", collectionId, e.getMessage()); + } + } + } catch (Exception e) { + LOG.warn("Error processing service {}: {}", serviceId, e.getMessage()); + } + } + + LOG.info("Fixed {} APICollection entities with incorrect FQN hash", fixedCount); + } + + /** + * Fix FQN and hash for APIEndpoint entities whose parent collection has a service with dots in + * its name. Must be called after fixApiCollectionFqnHash. + */ + public static void fixApiEndpointFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info("Starting migration to fix APIEndpoint FQN hash for services with dots in names"); + + // Find all API services with dots in their names + Set serviceIds = findServicesWithDotsInName(handle, "api_service_entity"); + if (serviceIds.isEmpty()) { + LOG.info("No API services with dots in names found. Skipping APIEndpoint FQN fix."); + return; + } + + // Find all API collections belonging to these services + Set collectionIds = + findChildEntityIds(collectionDAO, serviceIds, Entity.API_SERVICE, Entity.API_COLLECTION); + if (collectionIds.isEmpty()) { + LOG.info("No API collections found under services with dots. Skipping APIEndpoint FQN fix."); + return; + } + + // Find all API endpoints belonging to these collections + Set endpointIds = + findChildEntityIds( + collectionDAO, collectionIds, Entity.API_COLLECTION, Entity.API_ENDPOINT); + LOG.info("Found {} API endpoints to check", endpointIds.size()); + + int fixedCount = 0; + for (UUID endpointId : endpointIds) { + try { + APIEndpoint apiEndpoint = collectionDAO.apiEndpointDAO().findEntityById(endpointId); + if (apiEndpoint == null) continue; + + // Get the collection FQN + APICollection apiCollection = + collectionDAO.apiCollectionDAO().findEntityById(apiEndpoint.getApiCollection().getId()); + if (apiCollection == null) continue; + + String collectionFqn = apiCollection.getFullyQualifiedName(); + if (collectionFqn == null || !collectionFqn.contains("\"")) continue; + + String expectedFqn = FullyQualifiedName.add(collectionFqn, apiEndpoint.getName()); + String currentFqn = apiEndpoint.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing APIEndpoint FQN: {} -> {}", currentFqn, expectedFqn); + apiEndpoint.setFullyQualifiedName(expectedFqn); + collectionDAO.apiEndpointDAO().update(apiEndpoint); + fixedCount++; + } + } catch (Exception e) { + LOG.warn("Error processing APIEndpoint entity {}: {}", endpointId, e.getMessage()); + } + } + + LOG.info("Fixed {} APIEndpoint entities with incorrect FQN hash", fixedCount); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1120/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1120/MigrationUtil.java new file mode 100644 index 00000000000..1df85bfc0df --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1120/MigrationUtil.java @@ -0,0 +1,471 @@ +package org.openmetadata.service.migration.utils.v1120; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.core.Handle; +import org.openmetadata.schema.entity.data.APICollection; +import org.openmetadata.schema.entity.data.APIEndpoint; +import org.openmetadata.schema.entity.data.DashboardDataModel; +import org.openmetadata.schema.entity.data.Database; +import org.openmetadata.schema.entity.data.DatabaseSchema; +import org.openmetadata.schema.entity.data.StoredProcedure; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.ColumnUtil; +import org.openmetadata.service.util.FullyQualifiedName; + +/** + * Migration utility to fix FQN and FQN hash for entities whose parent service name contains dots. + * + *

Prior to this fix, the FQN was incorrectly built using getService().getName() instead of + * getService().getFullyQualifiedName(). This caused service names with dots (e.g., "my.service") + * to be incorrectly represented in child entity FQNs. + * + *

For example, if service FQN is "my.service" (quoted as "\"my.service\""), a Database named + * "mydb" would incorrectly have FQN "my.service.mydb" instead of the correct "\"my.service\".mydb" + */ +@Slf4j +public class MigrationUtil { + + private MigrationUtil() {} + + /** + * Find all service IDs that have dots in their names. These are the services whose child entities + * need FQN fixes. + */ + private static Set findServicesWithDotsInName(Handle handle, String serviceTable) { + Set serviceIds = new HashSet<>(); + String query = String.format("SELECT id FROM %s WHERE name LIKE '%%.%%'", serviceTable); + + try { + List> rows = handle.createQuery(query).mapToMap().list(); + for (Map row : rows) { + Object idObj = row.get("id"); + if (idObj != null) { + serviceIds.add(UUID.fromString(idObj.toString())); + } + } + } catch (Exception e) { + LOG.warn( + "Error finding services with dots in name from {}: {}", serviceTable, e.getMessage()); + } + + return serviceIds; + } + + /** Find entity IDs that belong to the given parent entities via relationship table. */ + private static Set findChildEntityIds( + CollectionDAO collectionDAO, Set parentIds, String fromType, String toType) { + Set childIds = new HashSet<>(); + for (UUID parentId : parentIds) { + List records = + collectionDAO + .relationshipDAO() + .findTo(parentId, fromType, Relationship.CONTAINS.ordinal(), toType); + for (CollectionDAO.EntityRelationshipRecord record : records) { + childIds.add(record.getId()); + } + } + return childIds; + } + + /** Fix FQN and hash for Database entities that have services with dots in their names. */ + public static void fixDatabaseFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info("Starting migration to fix Database FQN hash for services with dots in names"); + + Set serviceIds = findServicesWithDotsInName(handle, "dbservice_entity"); + if (serviceIds.isEmpty()) { + LOG.info("No database services with dots in names found. Skipping Database FQN fix."); + return; + } + LOG.info("Found {} database services with dots in names", serviceIds.size()); + + int fixedCount = 0; + for (UUID serviceId : serviceIds) { + try { + var service = collectionDAO.dbServiceDAO().findEntityById(serviceId); + if (service == null) continue; + + String serviceFqn = service.getFullyQualifiedName(); + if (serviceFqn == null || !serviceFqn.contains("\"")) continue; + + Set databaseIds = + findChildEntityIds( + collectionDAO, Set.of(serviceId), Entity.DATABASE_SERVICE, Entity.DATABASE); + + for (UUID databaseId : databaseIds) { + try { + Database database = collectionDAO.databaseDAO().findEntityById(databaseId); + if (database == null) continue; + + String expectedFqn = FullyQualifiedName.add(serviceFqn, database.getName()); + String currentFqn = database.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing Database FQN: {} -> {}", currentFqn, expectedFqn); + database.setFullyQualifiedName(expectedFqn); + collectionDAO.databaseDAO().update(database); + fixedCount++; + } + } catch (Exception e) { + LOG.warn("Error processing database entity {}: {}", databaseId, e.getMessage()); + } + } + } catch (Exception e) { + LOG.warn("Error processing service {}: {}", serviceId, e.getMessage()); + } + } + + LOG.info("Fixed {} Database entities with incorrect FQN hash", fixedCount); + } + + /** + * Fix FQN and hash for DatabaseSchema entities whose parent database has a service with dots in + * its name. Must be called after fixDatabaseFqnHash. + */ + public static void fixDatabaseSchemaFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info("Starting migration to fix DatabaseSchema FQN hash for services with dots in names"); + + Set serviceIds = findServicesWithDotsInName(handle, "dbservice_entity"); + if (serviceIds.isEmpty()) { + LOG.info("No database services with dots in names found. Skipping DatabaseSchema FQN fix."); + return; + } + + Set databaseIds = + findChildEntityIds(collectionDAO, serviceIds, Entity.DATABASE_SERVICE, Entity.DATABASE); + if (databaseIds.isEmpty()) { + LOG.info("No databases found under services with dots. Skipping DatabaseSchema FQN fix."); + return; + } + + Set schemaIds = + findChildEntityIds(collectionDAO, databaseIds, Entity.DATABASE, Entity.DATABASE_SCHEMA); + LOG.info("Found {} database schemas to check", schemaIds.size()); + + int fixedCount = 0; + for (UUID schemaId : schemaIds) { + try { + DatabaseSchema schema = collectionDAO.databaseSchemaDAO().findEntityById(schemaId); + if (schema == null) continue; + + Database database = + collectionDAO.databaseDAO().findEntityById(schema.getDatabase().getId()); + if (database == null) continue; + + String databaseFqn = database.getFullyQualifiedName(); + if (databaseFqn == null || !databaseFqn.contains("\"")) continue; + + String expectedFqn = FullyQualifiedName.add(databaseFqn, schema.getName()); + String currentFqn = schema.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing DatabaseSchema FQN: {} -> {}", currentFqn, expectedFqn); + schema.setFullyQualifiedName(expectedFqn); + collectionDAO.databaseSchemaDAO().update(schema); + fixedCount++; + } + } catch (Exception e) { + LOG.warn("Error processing DatabaseSchema entity {}: {}", schemaId, e.getMessage()); + } + } + + LOG.info("Fixed {} DatabaseSchema entities with incorrect FQN hash", fixedCount); + } + + /** + * Fix FQN and hash for Table entities whose parent schema has a service with dots in its name. + * Must be called after fixDatabaseSchemaFqnHash. + */ + public static void fixTableFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info("Starting migration to fix Table FQN hash for services with dots in names"); + + Set serviceIds = findServicesWithDotsInName(handle, "dbservice_entity"); + if (serviceIds.isEmpty()) { + LOG.info("No database services with dots in names found. Skipping Table FQN fix."); + return; + } + + Set databaseIds = + findChildEntityIds(collectionDAO, serviceIds, Entity.DATABASE_SERVICE, Entity.DATABASE); + if (databaseIds.isEmpty()) { + LOG.info("No databases found under services with dots. Skipping Table FQN fix."); + return; + } + + Set schemaIds = + findChildEntityIds(collectionDAO, databaseIds, Entity.DATABASE, Entity.DATABASE_SCHEMA); + if (schemaIds.isEmpty()) { + LOG.info("No schemas found under services with dots. Skipping Table FQN fix."); + return; + } + + Set tableIds = + findChildEntityIds(collectionDAO, schemaIds, Entity.DATABASE_SCHEMA, Entity.TABLE); + LOG.info("Found {} tables to check", tableIds.size()); + + int fixedCount = 0; + for (UUID tableId : tableIds) { + try { + Table table = collectionDAO.tableDAO().findEntityById(tableId); + if (table == null) continue; + + DatabaseSchema schema = + collectionDAO.databaseSchemaDAO().findEntityById(table.getDatabaseSchema().getId()); + if (schema == null) continue; + + String schemaFqn = schema.getFullyQualifiedName(); + if (schemaFqn == null || !schemaFqn.contains("\"")) continue; + + String expectedFqn = FullyQualifiedName.add(schemaFqn, table.getName()); + String currentFqn = table.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing Table FQN: {} -> {}", currentFqn, expectedFqn); + table.setFullyQualifiedName(expectedFqn); + ColumnUtil.setColumnFQN(expectedFqn, table.getColumns()); + collectionDAO.tableDAO().update(table); + fixedCount++; + } + } catch (Exception e) { + LOG.warn("Error processing Table entity {}: {}", tableId, e.getMessage()); + } + } + + LOG.info("Fixed {} Table entities with incorrect FQN hash", fixedCount); + } + + /** + * Fix FQN and hash for StoredProcedure entities whose parent schema has a service with dots in + * its name. Must be called after fixDatabaseSchemaFqnHash. + */ + public static void fixStoredProcedureFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info("Starting migration to fix StoredProcedure FQN hash for services with dots in names"); + + Set serviceIds = findServicesWithDotsInName(handle, "dbservice_entity"); + if (serviceIds.isEmpty()) { + LOG.info("No database services with dots in names found. Skipping StoredProcedure FQN fix."); + return; + } + + Set databaseIds = + findChildEntityIds(collectionDAO, serviceIds, Entity.DATABASE_SERVICE, Entity.DATABASE); + if (databaseIds.isEmpty()) { + LOG.info("No databases found under services with dots. Skipping StoredProcedure FQN fix."); + return; + } + + Set schemaIds = + findChildEntityIds(collectionDAO, databaseIds, Entity.DATABASE, Entity.DATABASE_SCHEMA); + if (schemaIds.isEmpty()) { + LOG.info("No schemas found under services with dots. Skipping StoredProcedure FQN fix."); + return; + } + + Set spIds = + findChildEntityIds( + collectionDAO, schemaIds, Entity.DATABASE_SCHEMA, Entity.STORED_PROCEDURE); + LOG.info("Found {} stored procedures to check", spIds.size()); + + int fixedCount = 0; + for (UUID spId : spIds) { + try { + StoredProcedure sp = collectionDAO.storedProcedureDAO().findEntityById(spId); + if (sp == null) continue; + + DatabaseSchema schema = + collectionDAO.databaseSchemaDAO().findEntityById(sp.getDatabaseSchema().getId()); + if (schema == null) continue; + + String schemaFqn = schema.getFullyQualifiedName(); + if (schemaFqn == null || !schemaFqn.contains("\"")) continue; + + String expectedFqn = FullyQualifiedName.add(schemaFqn, sp.getName()); + String currentFqn = sp.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing StoredProcedure FQN: {} -> {}", currentFqn, expectedFqn); + sp.setFullyQualifiedName(expectedFqn); + collectionDAO.storedProcedureDAO().update(sp); + fixedCount++; + } + } catch (Exception e) { + LOG.warn("Error processing StoredProcedure entity {}: {}", spId, e.getMessage()); + } + } + + LOG.info("Fixed {} StoredProcedure entities with incorrect FQN hash", fixedCount); + } + + /** + * Fix FQN and hash for DashboardDataModel entities that have services with dots in their names. + */ + public static void fixDashboardDataModelFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info( + "Starting migration to fix DashboardDataModel FQN hash for services with dots in names"); + + Set serviceIds = findServicesWithDotsInName(handle, "dashboard_service_entity"); + if (serviceIds.isEmpty()) { + LOG.info( + "No dashboard services with dots in names found. Skipping DashboardDataModel FQN fix."); + return; + } + LOG.info("Found {} dashboard services with dots in names", serviceIds.size()); + + int fixedCount = 0; + for (UUID serviceId : serviceIds) { + try { + var service = collectionDAO.dashboardServiceDAO().findEntityById(serviceId); + if (service == null) continue; + + String serviceFqn = service.getFullyQualifiedName(); + if (serviceFqn == null || !serviceFqn.contains("\"")) continue; + + Set dataModelIds = + findChildEntityIds( + collectionDAO, + Set.of(serviceId), + Entity.DASHBOARD_SERVICE, + Entity.DASHBOARD_DATA_MODEL); + + for (UUID dataModelId : dataModelIds) { + try { + DashboardDataModel dataModel = + collectionDAO.dashboardDataModelDAO().findEntityById(dataModelId); + if (dataModel == null) continue; + + String expectedFqn = FullyQualifiedName.add(serviceFqn + ".model", dataModel.getName()); + String currentFqn = dataModel.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing DashboardDataModel FQN: {} -> {}", currentFqn, expectedFqn); + dataModel.setFullyQualifiedName(expectedFqn); + ColumnUtil.setColumnFQN(expectedFqn, dataModel.getColumns()); + collectionDAO.dashboardDataModelDAO().update(dataModel); + fixedCount++; + } + } catch (Exception e) { + LOG.warn( + "Error processing DashboardDataModel entity {}: {}", dataModelId, e.getMessage()); + } + } + } catch (Exception e) { + LOG.warn("Error processing service {}: {}", serviceId, e.getMessage()); + } + } + + LOG.info("Fixed {} DashboardDataModel entities with incorrect FQN hash", fixedCount); + } + + /** Fix FQN and hash for APICollection entities that have services with dots in their names. */ + public static void fixApiCollectionFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info("Starting migration to fix APICollection FQN hash for services with dots in names"); + + Set serviceIds = findServicesWithDotsInName(handle, "api_service_entity"); + if (serviceIds.isEmpty()) { + LOG.info("No API services with dots in names found. Skipping APICollection FQN fix."); + return; + } + LOG.info("Found {} API services with dots in names", serviceIds.size()); + + int fixedCount = 0; + for (UUID serviceId : serviceIds) { + try { + var service = collectionDAO.apiServiceDAO().findEntityById(serviceId); + if (service == null) continue; + + String serviceFqn = service.getFullyQualifiedName(); + if (serviceFqn == null || !serviceFqn.contains("\"")) continue; + + Set collectionIds = + findChildEntityIds( + collectionDAO, Set.of(serviceId), Entity.API_SERVICE, Entity.API_COLLECTION); + + for (UUID collectionId : collectionIds) { + try { + APICollection apiCollection = + collectionDAO.apiCollectionDAO().findEntityById(collectionId); + if (apiCollection == null) continue; + + String expectedFqn = FullyQualifiedName.add(serviceFqn, apiCollection.getName()); + String currentFqn = apiCollection.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing APICollection FQN: {} -> {}", currentFqn, expectedFqn); + apiCollection.setFullyQualifiedName(expectedFqn); + collectionDAO.apiCollectionDAO().update(apiCollection); + fixedCount++; + } + } catch (Exception e) { + LOG.warn("Error processing APICollection entity {}: {}", collectionId, e.getMessage()); + } + } + } catch (Exception e) { + LOG.warn("Error processing service {}: {}", serviceId, e.getMessage()); + } + } + + LOG.info("Fixed {} APICollection entities with incorrect FQN hash", fixedCount); + } + + /** + * Fix FQN and hash for APIEndpoint entities whose parent collection has a service with dots in + * its name. Must be called after fixApiCollectionFqnHash. + */ + public static void fixApiEndpointFqnHash(Handle handle, CollectionDAO collectionDAO) { + LOG.info("Starting migration to fix APIEndpoint FQN hash for services with dots in names"); + + Set serviceIds = findServicesWithDotsInName(handle, "api_service_entity"); + if (serviceIds.isEmpty()) { + LOG.info("No API services with dots in names found. Skipping APIEndpoint FQN fix."); + return; + } + + Set collectionIds = + findChildEntityIds(collectionDAO, serviceIds, Entity.API_SERVICE, Entity.API_COLLECTION); + if (collectionIds.isEmpty()) { + LOG.info("No API collections found under services with dots. Skipping APIEndpoint FQN fix."); + return; + } + + Set endpointIds = + findChildEntityIds( + collectionDAO, collectionIds, Entity.API_COLLECTION, Entity.API_ENDPOINT); + LOG.info("Found {} API endpoints to check", endpointIds.size()); + + int fixedCount = 0; + for (UUID endpointId : endpointIds) { + try { + APIEndpoint apiEndpoint = collectionDAO.apiEndpointDAO().findEntityById(endpointId); + if (apiEndpoint == null) continue; + + APICollection apiCollection = + collectionDAO.apiCollectionDAO().findEntityById(apiEndpoint.getApiCollection().getId()); + if (apiCollection == null) continue; + + String collectionFqn = apiCollection.getFullyQualifiedName(); + if (collectionFqn == null || !collectionFqn.contains("\"")) continue; + + String expectedFqn = FullyQualifiedName.add(collectionFqn, apiEndpoint.getName()); + String currentFqn = apiEndpoint.getFullyQualifiedName(); + + if (!expectedFqn.equals(currentFqn)) { + LOG.debug("Fixing APIEndpoint FQN: {} -> {}", currentFqn, expectedFqn); + apiEndpoint.setFullyQualifiedName(expectedFqn); + collectionDAO.apiEndpointDAO().update(apiEndpoint); + fixedCount++; + } + } catch (Exception e) { + LOG.warn("Error processing APIEndpoint entity {}: {}", endpointId, e.getMessage()); + } + } + + LOG.info("Fixed {} APIEndpoint entities with incorrect FQN hash", fixedCount); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/migration/v1120/MigrationUtilTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/migration/v1120/MigrationUtilTest.java new file mode 100644 index 00000000000..dd7a94ee729 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/migration/v1120/MigrationUtilTest.java @@ -0,0 +1,363 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.migration.v1120; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS; + +import java.io.IOException; +import java.util.UUID; +import org.jdbi.v3.core.Handle; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestMethodOrder; +import org.openmetadata.schema.api.data.CreateDatabase; +import org.openmetadata.schema.api.services.CreateDatabaseService; +import org.openmetadata.schema.entity.data.Database; +import org.openmetadata.schema.entity.data.DatabaseSchema; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.services.DatabaseService; +import org.openmetadata.service.Entity; +import org.openmetadata.service.OpenMetadataApplicationTest; +import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.migration.utils.v1120.MigrationUtil; +import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.TestUtils; + +/** + * Tests for the v1.12.0 migration that fixes FQN hash for entities whose service name contains + * dots. + * + *

The bug was in repository code that used service.getName() instead of + * service.getFullyQualifiedName() when building child entity FQNs. For services with dots in their + * names (e.g., "my.service"), this resulted in incorrect FQNs: + * + *

+ * + *

This test simulates the bug by directly inserting entities with bad FQNs into the database, + * then verifies that the migration correctly fixes them. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class MigrationUtilTest extends OpenMetadataApplicationTest { + + private CollectionDAO collectionDAO; + + private DatabaseService serviceWithDots; + private String databaseName; + private String schemaName; + private String tableName; + + private UUID databaseId; + private UUID schemaId; + private UUID tableId; + + // Bad FQNs (as if created with buggy service.getName() code) + private String badDatabaseFqn; + private String badSchemaFqn; + private String badTableFqn; + + // Expected correct FQNs (using service.getFullyQualifiedName()) + private String expectedDatabaseFqn; + private String expectedSchemaFqn; + private String expectedTableFqn; + + @BeforeAll + public void setup(TestInfo test) throws IOException { + collectionDAO = Entity.getCollectionDAO(); + + // Create a database service with dots in its name + String serviceName = "test.service.with.dots." + UUID.randomUUID().toString().substring(0, 8); + CreateDatabaseService createService = + new CreateDatabaseService() + .withName(serviceName) + .withServiceType(CreateDatabaseService.DatabaseServiceType.Mysql) + .withConnection(TestUtils.MYSQL_DATABASE_CONNECTION); + + serviceWithDots = + TestUtils.post( + getResource("services/databaseServices"), + createService, + DatabaseService.class, + ADMIN_AUTH_HEADERS); + + // Verify service FQN contains quotes due to dots + assertTrue( + serviceWithDots.getFullyQualifiedName().contains("\""), + "Service FQN should contain quotes: " + serviceWithDots.getFullyQualifiedName()); + + // Generate unique names for test entities + databaseName = "testdb" + UUID.randomUUID().toString().substring(0, 8); + schemaName = "testschema" + UUID.randomUUID().toString().substring(0, 8); + tableName = "testtable" + UUID.randomUUID().toString().substring(0, 8); + + // Compute the BAD FQNs (using service.getName() - the buggy behavior) + // This is what entities would have if created with the buggy code + badDatabaseFqn = serviceWithDots.getName() + "." + databaseName; + badSchemaFqn = badDatabaseFqn + "." + schemaName; + badTableFqn = badSchemaFqn + "." + tableName; + + // Compute the CORRECT FQNs (using service.getFullyQualifiedName()) + expectedDatabaseFqn = + FullyQualifiedName.add(serviceWithDots.getFullyQualifiedName(), databaseName); + expectedSchemaFqn = FullyQualifiedName.add(expectedDatabaseFqn, schemaName); + expectedTableFqn = FullyQualifiedName.add(expectedSchemaFqn, tableName); + } + + @Test + @Order(1) + void testInsertEntitiesWithBadFqn() { + // Simulate the bug by inserting entities with bad FQNs directly into the database. + // This mimics what would have happened if entities were created with the buggy code + // that used service.getName() instead of service.getFullyQualifiedName() + + databaseId = UUID.randomUUID(); + schemaId = UUID.randomUUID(); + tableId = UUID.randomUUID(); + + try (Handle handle = jdbi.open()) { + // Insert database with bad FQN + // Note: id, name, updatedAt, updatedBy, deleted are GENERATED columns derived from json + // We only need to insert json and fqnHash + String databaseJson = + String.format( + "{\"id\":\"%s\",\"name\":\"%s\",\"fullyQualifiedName\":\"%s\"," + + "\"service\":{\"id\":\"%s\",\"type\":\"databaseService\"}," + + "\"serviceType\":\"Mysql\",\"version\":0.1,\"updatedAt\":%d,\"updatedBy\":\"admin\"," + + "\"deleted\":false}", + databaseId, + databaseName, + badDatabaseFqn, + serviceWithDots.getId(), + System.currentTimeMillis()); + + handle.execute( + String.format( + "INSERT INTO database_entity (json, fqnHash) VALUES ('%s', '%s')", + databaseJson, FullyQualifiedName.buildHash(badDatabaseFqn))); + + // Insert relationship: service -> database + handle.execute( + String.format( + "INSERT INTO entity_relationship (fromId, toId, fromEntity, toEntity, relation) " + + "VALUES ('%s', '%s', '%s', '%s', %d)", + serviceWithDots.getId(), + databaseId, + Entity.DATABASE_SERVICE, + Entity.DATABASE, + 0)); // CONTAINS relationship + + // Insert schema with bad FQN + String schemaJson = + String.format( + "{\"id\":\"%s\",\"name\":\"%s\",\"fullyQualifiedName\":\"%s\"," + + "\"database\":{\"id\":\"%s\",\"type\":\"database\"}," + + "\"service\":{\"id\":\"%s\",\"type\":\"databaseService\"}," + + "\"serviceType\":\"Mysql\",\"version\":0.1,\"updatedAt\":%d,\"updatedBy\":\"admin\"," + + "\"deleted\":false}", + schemaId, + schemaName, + badSchemaFqn, + databaseId, + serviceWithDots.getId(), + System.currentTimeMillis()); + + handle.execute( + String.format( + "INSERT INTO database_schema_entity (json, fqnHash) VALUES ('%s', '%s')", + schemaJson, FullyQualifiedName.buildHash(badSchemaFqn))); + + // Insert relationship: database -> schema + handle.execute( + String.format( + "INSERT INTO entity_relationship (fromId, toId, fromEntity, toEntity, relation) " + + "VALUES ('%s', '%s', '%s', '%s', %d)", + databaseId, schemaId, Entity.DATABASE, Entity.DATABASE_SCHEMA, 0)); + + // Insert table with bad FQN + String tableJson = + String.format( + "{\"id\":\"%s\",\"name\":\"%s\",\"fullyQualifiedName\":\"%s\"," + + "\"columns\":[{\"name\":\"id\",\"dataType\":\"INT\",\"fullyQualifiedName\":\"%s.id\"}]," + + "\"databaseSchema\":{\"id\":\"%s\",\"type\":\"databaseSchema\"}," + + "\"database\":{\"id\":\"%s\",\"type\":\"database\"}," + + "\"service\":{\"id\":\"%s\",\"type\":\"databaseService\"}," + + "\"serviceType\":\"Mysql\",\"tableType\":\"Regular\"," + + "\"version\":0.1,\"updatedAt\":%d,\"updatedBy\":\"admin\",\"deleted\":false}", + tableId, + tableName, + badTableFqn, + badTableFqn, + schemaId, + databaseId, + serviceWithDots.getId(), + System.currentTimeMillis()); + + handle.execute( + String.format( + "INSERT INTO table_entity (json, fqnHash) VALUES ('%s', '%s')", + tableJson, FullyQualifiedName.buildHash(badTableFqn))); + + // Insert relationship: schema -> table + handle.execute( + String.format( + "INSERT INTO entity_relationship (fromId, toId, fromEntity, toEntity, relation) " + + "VALUES ('%s', '%s', '%s', '%s', %d)", + schemaId, tableId, Entity.DATABASE_SCHEMA, Entity.TABLE, 0)); + } + + // Verify entities were inserted with bad FQNs (no quotes in FQN) + Database insertedDatabase = collectionDAO.databaseDAO().findEntityById(databaseId); + assertEquals(badDatabaseFqn, insertedDatabase.getFullyQualifiedName()); + assertFalse( + insertedDatabase.getFullyQualifiedName().contains("\""), + "Bad FQN should NOT contain quotes: " + insertedDatabase.getFullyQualifiedName()); + + DatabaseSchema insertedSchema = collectionDAO.databaseSchemaDAO().findEntityById(schemaId); + assertEquals(badSchemaFqn, insertedSchema.getFullyQualifiedName()); + assertFalse( + insertedSchema.getFullyQualifiedName().contains("\""), + "Bad FQN should NOT contain quotes: " + insertedSchema.getFullyQualifiedName()); + + Table insertedTable = collectionDAO.tableDAO().findEntityById(tableId); + assertEquals(badTableFqn, insertedTable.getFullyQualifiedName()); + assertFalse( + insertedTable.getFullyQualifiedName().contains("\""), + "Bad FQN should NOT contain quotes: " + insertedTable.getFullyQualifiedName()); + } + + @Test + @Order(2) + void testMigrationFixesBadFqns() { + // Run the migration to fix the bad FQNs + try (Handle handle = jdbi.open()) { + // Fix in order: Database -> Schema -> Table (parent before child) + MigrationUtil.fixDatabaseFqnHash(handle, collectionDAO); + MigrationUtil.fixDatabaseSchemaFqnHash(handle, collectionDAO); + MigrationUtil.fixTableFqnHash(handle, collectionDAO); + } + + // Verify FQNs are now correct (with quotes) + Database fixedDatabase = collectionDAO.databaseDAO().findEntityById(databaseId); + assertEquals( + expectedDatabaseFqn, fixedDatabase.getFullyQualifiedName(), "Database FQN should be fixed"); + assertTrue( + fixedDatabase.getFullyQualifiedName().contains("\""), + "Fixed Database FQN should contain quotes: " + fixedDatabase.getFullyQualifiedName()); + + DatabaseSchema fixedSchema = collectionDAO.databaseSchemaDAO().findEntityById(schemaId); + assertEquals( + expectedSchemaFqn, fixedSchema.getFullyQualifiedName(), "Schema FQN should be fixed"); + assertTrue( + fixedSchema.getFullyQualifiedName().contains("\""), + "Fixed Schema FQN should contain quotes: " + fixedSchema.getFullyQualifiedName()); + + Table fixedTable = collectionDAO.tableDAO().findEntityById(tableId); + assertEquals(expectedTableFqn, fixedTable.getFullyQualifiedName(), "Table FQN should be fixed"); + assertTrue( + fixedTable.getFullyQualifiedName().contains("\""), + "Fixed Table FQN should contain quotes: " + fixedTable.getFullyQualifiedName()); + + // Also verify column FQN was fixed + assertEquals( + expectedTableFqn + ".id", + fixedTable.getColumns().get(0).getFullyQualifiedName(), + "Column FQN should be fixed"); + } + + @Test + @Order(3) + void testMigrationIsIdempotent() { + // Get current FQNs after first migration + Database beforeDatabase = collectionDAO.databaseDAO().findEntityById(databaseId); + DatabaseSchema beforeSchema = collectionDAO.databaseSchemaDAO().findEntityById(schemaId); + Table beforeTable = collectionDAO.tableDAO().findEntityById(tableId); + + // Run migration again - should not change anything + try (Handle handle = jdbi.open()) { + MigrationUtil.fixDatabaseFqnHash(handle, collectionDAO); + MigrationUtil.fixDatabaseSchemaFqnHash(handle, collectionDAO); + MigrationUtil.fixTableFqnHash(handle, collectionDAO); + } + + // Verify FQNs haven't changed + Database afterDatabase = collectionDAO.databaseDAO().findEntityById(databaseId); + DatabaseSchema afterSchema = collectionDAO.databaseSchemaDAO().findEntityById(schemaId); + Table afterTable = collectionDAO.tableDAO().findEntityById(tableId); + + assertEquals( + beforeDatabase.getFullyQualifiedName(), + afterDatabase.getFullyQualifiedName(), + "Database FQN should not change on re-run"); + assertEquals( + beforeSchema.getFullyQualifiedName(), + afterSchema.getFullyQualifiedName(), + "Schema FQN should not change on re-run"); + assertEquals( + beforeTable.getFullyQualifiedName(), + afterTable.getFullyQualifiedName(), + "Table FQN should not change on re-run"); + } + + @Test + @Order(4) + void testMigrationSkipsServicesWithoutDots() throws IOException { + // Create a service without dots in its name + String normalServiceName = "normalservice" + UUID.randomUUID().toString().substring(0, 8); + CreateDatabaseService createService = + new CreateDatabaseService() + .withName(normalServiceName) + .withServiceType(CreateDatabaseService.DatabaseServiceType.Mysql) + .withConnection(TestUtils.MYSQL_DATABASE_CONNECTION); + + DatabaseService normalService = + TestUtils.post( + getResource("services/databaseServices"), + createService, + DatabaseService.class, + ADMIN_AUTH_HEADERS); + + // Create a database under this normal service (using API, which has fixed code) + CreateDatabase createDatabase = + new CreateDatabase() + .withName("normaldb" + UUID.randomUUID().toString().substring(0, 8)) + .withService(normalService.getFullyQualifiedName()); + Database normalDatabase = + TestUtils.post( + getResource("databases"), createDatabase, Database.class, ADMIN_AUTH_HEADERS); + + String originalFqn = normalDatabase.getFullyQualifiedName(); + + // Run migration + try (Handle handle = jdbi.open()) { + MigrationUtil.fixDatabaseFqnHash(handle, collectionDAO); + } + + // Verify FQN hasn't changed (no dots in service name, so migration skips it) + Database afterMigration = collectionDAO.databaseDAO().findEntityById(normalDatabase.getId()); + assertEquals( + originalFqn, + afterMigration.getFullyQualifiedName(), + "Database under service without dots should not be modified"); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index 08a97ac3e26..6efa2c4f523 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -2263,6 +2263,91 @@ public abstract class EntityResourceTestSubclasses that support containers with dots in their name should override + * createContainerWithDotsInName() to return the container reference. + */ + @Test + @Execution(ExecutionMode.CONCURRENT) + protected void post_entityUnderContainerWithDots_200() throws IOException { + // Get container with dots in name - subclasses should override this method + EntityReference containerWithDots = createContainerWithDotsInName(entityType + ".service.test"); + if (containerWithDots == null) { + return; // Entity doesn't support containers with dots or this test + } + + try { + // Create an entity under the container with dots in name + String entityName = entityType + "_under_dotted_container"; + K request = createRequestUnderContainer(entityName, containerWithDots); + if (request == null) { + return; // Entity doesn't support creating under a different container + } + + T entity = createEntity(request, ADMIN_AUTH_HEADERS); + + // Verify FQN contains the quoted container name + String fqn = entity.getFullyQualifiedName(); + assertTrue( + fqn.contains("\""), + "FQN should contain quoted container name when container has dots: " + fqn); + + // Verify the entity can be retrieved by name + T retrieved = getEntityByName(fqn, "", ADMIN_AUTH_HEADERS); + assertEquals(entity.getId(), retrieved.getId()); + + // Verify listing by service/container works correctly + Map queryParams = new HashMap<>(); + queryParams.put("service", containerWithDots.getFullyQualifiedName()); + ResultList list = listEntities(queryParams, ADMIN_AUTH_HEADERS); + assertFalse( + list.getData().isEmpty(), + "Should find entities when filtering by container with dots in name"); + assertTrue( + list.getData().stream().anyMatch(e -> e.getId().equals(entity.getId())), + "Should find the created entity in the list"); + } finally { + // Cleanup: delete the container with dots + deleteContainerWithDotsInName(containerWithDots); + } + } + + /** + * Override this method in subclasses to create a container (service) with dots in its name. For + * example, DatabaseResourceTest would create a DatabaseService with a name like "my.service.test" + * + * @param name the name to use for the container (will contain dots) + * @return the EntityReference to the created container, or null if not supported + */ + protected EntityReference createContainerWithDotsInName(String name) throws IOException { + return null; // Default implementation - subclasses override + } + + /** + * Override this method in subclasses to create a request for an entity under the given container. + * + * @param name the name for the new entity + * @param container the container reference with dots in its name + * @return the create request, or null if not supported + */ + protected K createRequestUnderContainer(String name, EntityReference container) { + return null; // Default implementation - subclasses override + } + + /** + * Override this method in subclasses to delete the container created by + * createContainerWithDotsInName. + * + * @param container the container to delete + */ + protected void deleteContainerWithDotsInName(EntityReference container) throws IOException { + // Default implementation - subclasses override + } + ////////////////////////////////////////////////////////////////////////////////////////////////// // Common entity tests for PUT operations ////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseResourceTest.java index 3e3fdeb06ff..12c781edcab 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/DatabaseResourceTest.java @@ -417,7 +417,7 @@ public class DatabaseResourceTest extends EntityResourceTest authHeaders) { assertReference(expected.getService(), updated.getService()); assertEquals( - FullyQualifiedName.build(updated.getService().getName(), updated.getName()), + FullyQualifiedName.add(updated.getService().getFullyQualifiedName(), updated.getName()), updated.getFullyQualifiedName()); } + @Override + protected EntityReference createContainerWithDotsInName(String name) throws IOException { + DatabaseServiceResourceTest serviceTest = new DatabaseServiceResourceTest(); + CreateDatabaseService createService = serviceTest.createRequest(name); + DatabaseService service = serviceTest.createEntity(createService, ADMIN_AUTH_HEADERS); + return service.getEntityReference(); + } + + @Override + protected CreateDatabase createRequestUnderContainer(String name, EntityReference container) { + return new CreateDatabase().withName(name).withService(container.getFullyQualifiedName()); + } + + @Override + protected void deleteContainerWithDotsInName(EntityReference container) throws IOException { + DatabaseServiceResourceTest serviceTest = new DatabaseServiceResourceTest(); + serviceTest.deleteEntity(container.getId(), true, true, ADMIN_AUTH_HEADERS); + } + @Override public void assertFieldChange(String fieldName, Object expected, Object actual) { if (fieldName.endsWith("owners") && (expected != null && actual != null)) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java index e0284659094..4602d33efc1 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResourceTest.java @@ -116,6 +116,45 @@ public class DashboardDataModelResourceTest } } + @Test + void post_dataModelWithServiceNameContainingDots_200_ok(TestInfo test) throws IOException { + // Create a dashboard service with dots in its name + DashboardServiceResourceTest serviceTest = new DashboardServiceResourceTest(); + String serviceNameWithDots = "service.with.dots." + test.getDisplayName(); + CreateDashboardService createService = serviceTest.createRequest(serviceNameWithDots); + DashboardService service = serviceTest.createEntity(createService, ADMIN_AUTH_HEADERS); + + // Create a data model under the service with dots in its name + CreateDashboardDataModel createDataModel = + createRequest(test.getDisplayName() + "_datamodel") + .withService(service.getFullyQualifiedName()); + DashboardDataModel dataModel = createAndCheckEntity(createDataModel, ADMIN_AUTH_HEADERS); + + // Verify we can list data models by filtering on service name containing dots + Map queryParams = new HashMap<>(); + queryParams.put("service", service.getFullyQualifiedName()); + ResultList list = listEntities(queryParams, ADMIN_AUTH_HEADERS); + + // Should find at least one data model + assertFalse(list.getData().isEmpty(), "Should find data models for service with dots in name"); + + // Find our created data model in the list + boolean found = list.getData().stream().anyMatch(dm -> dm.getId().equals(dataModel.getId())); + assertTrue( + found, "Should find the created data model when filtering by service name with dots"); + + // Verify all returned data models belong to the correct service + for (DashboardDataModel dm : list.getData()) { + assertEquals( + service.getFullyQualifiedName(), + dm.getService().getFullyQualifiedName(), + "All data models should belong to the service with dots in name"); + } + + // Cleanup + serviceTest.deleteEntity(service.getId(), true, true, ADMIN_AUTH_HEADERS); + } + @Test void test_mutuallyExclusiveTags(TestInfo testInfo) { CreateDashboardDataModel create = @@ -250,6 +289,32 @@ public class DashboardDataModelResourceTest return entity.getService(); } + @Override + protected EntityReference createContainerWithDotsInName(String name) throws IOException { + DashboardServiceResourceTest serviceTest = new DashboardServiceResourceTest(); + CreateDashboardService createService = serviceTest.createRequest(name); + DashboardService service = serviceTest.createEntity(createService, ADMIN_AUTH_HEADERS); + return service.getEntityReference(); + } + + @Override + protected CreateDashboardDataModel createRequestUnderContainer( + String name, EntityReference container) { + return new CreateDashboardDataModel() + .withName(name) + .withService(container.getFullyQualifiedName()) + .withServiceType(CreateDashboardDataModel.DashboardServiceType.Metabase) + .withSql("SELECT * FROM tab1;") + .withDataModelType(DataModelType.MetabaseDataModel) + .withColumns(COLUMNS); + } + + @Override + protected void deleteContainerWithDotsInName(EntityReference container) throws IOException { + DashboardServiceResourceTest serviceTest = new DashboardServiceResourceTest(); + serviceTest.deleteEntity(container.getId(), true, true, ADMIN_AUTH_HEADERS); + } + @Override public void validateCreatedEntity( DashboardDataModel dashboardDataModel, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Dashboards.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Dashboards.spec.ts index cd53074dea1..165d1f0d9cc 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Dashboards.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/Dashboards.spec.ts @@ -16,7 +16,11 @@ import { DashboardClass } from '../../support/entity/DashboardClass'; import { EntityTypeEndpoint } from '../../support/entity/Entity.interface'; import { DashboardServiceClass } from '../../support/entity/service/DashboardServiceClass'; import { performAdminLogin } from '../../utils/admin'; -import { redirectToHomePage, toastNotification } from '../../utils/common'; +import { + redirectToHomePage, + toastNotification, + uuid, +} from '../../utils/common'; import { assignTagToChildren, generateEntityChildren, @@ -229,3 +233,108 @@ test.describe('Data Model', () => { }); }); }); + +test.describe('Data Model with special characters in name', () => { + const uniqueId = uuid(); + const serviceNameWithDot = `pw.dashboard.service-${uniqueId}`; + const dataModelName = `pw-data-model-${uniqueId}`; + let serviceResponseData: { fullyQualifiedName: string }; + let dataModelResponseData: { fullyQualifiedName: string; name: string }; + + test.beforeAll('Setup pre-requests', async ({ browser }) => { + const { apiContext, afterAction } = await performAdminLogin(browser); + + const serviceResponse = await apiContext.post( + '/api/v1/services/dashboardServices', + { + data: { + name: serviceNameWithDot, + serviceType: 'PowerBI', + connection: { + config: { + type: 'PowerBI', + clientId: 'test-client-id', + clientSecret: 'test-client-secret', + tenantId: 'test-tenant-id', + }, + }, + }, + } + ); + + expect(serviceResponse.ok()).toBeTruthy(); + + serviceResponseData = await serviceResponse.json(); + + const dataModelResponse = await apiContext.post( + '/api/v1/dashboard/datamodels', + { + data: { + name: dataModelName, + displayName: dataModelName, + description: `Data model for service with dots in name`, + service: serviceNameWithDot, + columns: [ + { + name: 'column_1', + dataType: 'VARCHAR', + dataLength: 256, + dataTypeDisplay: 'varchar', + description: 'Test column', + }, + ], + dataModelType: 'PowerBIDataModel', + }, + } + ); + + expect(dataModelResponse.ok()).toBeTruthy(); + + dataModelResponseData = await dataModelResponse.json(); + + await afterAction(); + }); + + test.afterAll('Clean up', async ({ browser }) => { + const { afterAction, apiContext } = await performAdminLogin(browser); + + await apiContext.delete( + `/api/v1/services/dashboardServices/name/${encodeURIComponent( + serviceResponseData.fullyQualifiedName + )}?recursive=true&hardDelete=true` + ); + + await afterAction(); + }); + + test('should display data model when service name contains dots', async ({ + page, + }) => { + await page.goto( + `/service/dashboardServices/${encodeURIComponent( + serviceNameWithDot + )}/data-model` + ); + + await page.waitForLoadState('networkidle'); + await page.waitForSelector('[data-testid="loader"]', { state: 'hidden' }); + + await page.waitForSelector('.ant-spin', { + state: 'detached', + }); + + const dataModelLink = page.getByTestId( + `data-model-${dataModelResponseData.name}` + ); + + await expect(dataModelLink).toBeVisible(); + await expect(dataModelLink).toHaveText(dataModelResponseData.name); + + await dataModelLink.click(); + await page.waitForLoadState('networkidle'); + + await expect(page.getByTestId('entity-header-name')).toContainText( + dataModelName + ); + }); +});