yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-07 21:16:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Failing TestSuite Migration (#12730)

Browse Source
This commit is contained in:
Mohit Yadav 2023-08-03 17:06:30 +05:30 committed by GitHub
parent 403e407478
commit daf6012374
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 80 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java
View File

@ -257,6 +257,10 @@ public interface EntityDAO<T extends EntityInterface> {
insert(getTableName(), getNameHashColumn(), fqn, JsonUtils.pojoToJson(entity));
}
default void insert(String nameHash, EntityInterface entity, String fqn) throws JsonProcessingException {
insert(getTableName(), nameHash, fqn, JsonUtils.pojoToJson(entity));
}
default void update(UUID id, String fqn, String json) {
update(getTableName(), getNameHashColumn(), fqn, id.toString(), json);
}
@ -270,6 +274,15 @@ public interface EntityDAO<T extends EntityInterface> {
JsonUtils.pojoToJson(entity));
}
default void update(String nameHashColumn, EntityInterface entity) throws JsonProcessingException {
update(
getTableName(),
nameHashColumn,
entity.getFullyQualifiedName(),
entity.getId().toString(),
JsonUtils.pojoToJson(entity));
}
default String getCondition(Include include) {
if (!supportsSoftDelete()) {
return "";
@ -298,6 +311,11 @@ public interface EntityDAO<T extends EntityInterface> {
return jsonToEntity(findByName(getTableName(), getNameHashColumn(), fqn, getCondition(include)), fqn);
}
@SneakyThrows
default T findEntityByName(String fqn, String nameHashColumn, Include include) {
return jsonToEntity(findByName(getTableName(), nameHashColumn, fqn, getCondition(include)), fqn);
}
default T jsonToEntity(String json, String identity) throws IOException {
Class<T> clz = getEntityClass();
T entity = json != null ? JsonUtils.readValue(json, clz) : null;

103
openmetadata-service/src/main/java/org/openmetadata/service/migration/versions/utils/v110/MigrationUtil.java
View File

@ -101,10 +101,16 @@ public class MigrationUtil {
@SneakyThrows
public static <T extends EntityInterface> void updateFQNHashForEntity(
Handle handle, Class<T> clazz, EntityDAO<T> dao, int limitParam) {
String nameHashColumn = dao.getNameHashColumn();
if (dao instanceof CollectionDAO.TestSuiteDAO) {
// We have to do this since this column in changed in the dao in latest version after this , and this will fail
// the migrations here
nameHashColumn = "nameHash";
}
if (Boolean.TRUE.equals(DatasourceConfig.getInstance().isMySQL())) {
readAndProcessEntity(
handle,
String.format(MYSQL_ENTITY_UPDATE, dao.getTableName(), dao.getNameHashColumn()),
String.format(MYSQL_ENTITY_UPDATE, dao.getTableName(), nameHashColumn),
clazz,
dao,
false,
@ -112,7 +118,7 @@ public class MigrationUtil {
} else {
readAndProcessEntity(
handle,
String.format(POSTGRES_ENTITY_UPDATE, dao.getTableName(), dao.getNameHashColumn()),
String.format(POSTGRES_ENTITY_UPDATE, dao.getTableName(), nameHashColumn),
clazz,
dao,
false,
@ -123,18 +129,19 @@ public class MigrationUtil {
@SneakyThrows
public static <T extends EntityInterface> void updateFQNHashForEntityWithName(
Handle handle, Class<T> clazz, EntityDAO<T> dao, int limitParam) {
String nameHashColumn = dao.getNameHashColumn();
if (dao instanceof CollectionDAO.TestSuiteDAO) {
// We have to do this since this column in changed in the dao in latest version after this , and this will fail
// the migrations here
nameHashColumn = "nameHash";
}
if (Boolean.TRUE.equals(DatasourceConfig.getInstance().isMySQL())) {
readAndProcessEntity(
handle,
String.format(MYSQL_ENTITY_UPDATE, dao.getTableName(), dao.getNameHashColumn()),
clazz,
dao,
true,
limitParam);
handle, String.format(MYSQL_ENTITY_UPDATE, dao.getTableName(), nameHashColumn), clazz, dao, true, limitParam);
} else {
readAndProcessEntity(
handle,
String.format(POSTGRES_ENTITY_UPDATE, dao.getTableName(), dao.getNameHashColumn()),
String.format(POSTGRES_ENTITY_UPDATE, dao.getTableName(), nameHashColumn),
clazz,
dao,
true,
@ -450,12 +457,16 @@ public class MigrationUtil {
TestSuite stored;
try {
// If entity is found by Hash it is already migrated
testSuiteRepository.getDao().findEntityByName(EntityInterfaceUtil.quoteName(testSuiteFqn), Include.ALL);
testSuiteRepository
.getDao()
.findEntityByName(EntityInterfaceUtil.quoteName(testSuiteFqn), "nameHash", Include.ALL);
} catch (EntityNotFoundException entityNotFoundException) {
try {
// Check if the test Suite Exists, this brings the data on nameHash basis
stored =
testSuiteRepository.getDao().findEntityByName(EntityInterfaceUtil.quoteName(testSuiteFqn), Include.ALL);
testSuiteRepository
.getDao()
.findEntityByName(EntityInterfaceUtil.quoteName(testSuiteFqn), "nameHash", Include.ALL);
testSuiteRepository.addRelationship(
stored.getId(), test.getId(), TEST_SUITE, TEST_CASE, Relationship.CONTAINS);
stored.setExecutable(true);
@ -465,38 +476,48 @@ public class MigrationUtil {
stored.setDisplayName(testSuiteFqn);
testSuiteRepository.getDao().update(stored);
} catch (EntityNotFoundException ex) {
TestSuite newExecutableTestSuite =
getTestSuite(
collectionDAO,
new CreateTestSuite()
.withName(FullyQualifiedName.buildHash(testSuiteFqn))
.withDisplayName(testSuiteFqn)
.withExecutableEntityReference(entityLink.getEntityFQN()),
"ingestion-bot")
.withExecutable(false);
// Create
testSuiteRepository.prepareInternal(newExecutableTestSuite);
testSuiteRepository.getDao().insert(newExecutableTestSuite, newExecutableTestSuite.getFullyQualifiedName());
// Here we aer manually adding executable relationship since the table Repository is not registered and result
// into null for entity type table
testSuiteRepository.addRelationship(
newExecutableTestSuite.getExecutableEntityReference().getId(),
newExecutableTestSuite.getId(),
Entity.TABLE,
TEST_SUITE,
Relationship.CONTAINS);
try {
TestSuite newExecutableTestSuite =
getTestSuite(
collectionDAO,
new CreateTestSuite()
.withName(FullyQualifiedName.buildHash(testSuiteFqn))
.withDisplayName(testSuiteFqn)
.withExecutableEntityReference(entityLink.getEntityFQN()),
"ingestion-bot")
.withExecutable(false);
// Create
testSuiteRepository.prepareInternal(newExecutableTestSuite);
testSuiteRepository
.getDao()
.insert("nameHash", newExecutableTestSuite, newExecutableTestSuite.getFullyQualifiedName());
// Here we aer manually adding executable relationship since the table Repository is not registered and
// result
// into null for entity type table
testSuiteRepository.addRelationship(
newExecutableTestSuite.getExecutableEntityReference().getId(),
newExecutableTestSuite.getId(),
Entity.TABLE,
TEST_SUITE,
Relationship.CONTAINS);
// add relationship from testSuite to TestCases
testSuiteRepository.addRelationship(
newExecutableTestSuite.getId(), test.getId(), TEST_SUITE, TEST_CASE, Relationship.CONTAINS);
// add relationship from testSuite to TestCases
testSuiteRepository.addRelationship(
newExecutableTestSuite.getId(), test.getId(), TEST_SUITE, TEST_CASE, Relationship.CONTAINS);
// Not a good approach but executable cannot be set true before
TestSuite temp =
testSuiteRepository
.getDao()
.findEntityByName(EntityInterfaceUtil.quoteName(FullyQualifiedName.buildHash(testSuiteFqn)));
temp.setExecutable(true);
testSuiteRepository.getDao().update(temp);
// Not a good approach but executable cannot be set true before
TestSuite temp =
testSuiteRepository
.getDao()
.findEntityByName(
EntityInterfaceUtil.quoteName(FullyQualifiedName.buildHash(testSuiteFqn)),
"nameHash",
Include.ALL);
temp.setExecutable(true);
testSuiteRepository.getDao().update("nameHash", temp);
} catch (Exception exIgnore) {
LOG.warn("Ignoring error since already added: {}", ex.getMessage());
}
}
}
}