yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-07 13:07:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update algorithm and server as major change (#1505)

Browse Source
This commit is contained in:
Pere Miquel Brull 2021-12-01 19:59:13 +01:00 committed by GitHub
parent 0e205d93dd
commit 2171236ada
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 80 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/MlModelRepository.java
View File

@ -342,15 +342,19 @@ public class MlModelRepository extends EntityRepository<MlModel> {
}
private void updateAlgorithm(MlModel origModel, MlModel updatedModel) throws JsonProcessingException {
recordChange("algorithm", origModel.getAlgorithm(), updatedModel.getAlgorithm());
// Updating an algorithm should be flagged for an ML Model
if (recordChange("algorithm", origModel.getAlgorithm(), updatedModel.getAlgorithm())) {
// Mark the EntityUpdater version change to major
majorVersionChange = true;
}
}
private void updateMlFeatures(MlModel origModel, MlModel updatedModel) throws JsonProcessingException {
recordChange("mlFeatures", origModel.getMlFeatures(), updatedModel.getMlFeatures());
recordChange("mlFeatures", origModel.getMlFeatures(), updatedModel.getMlFeatures(), true);
}
private void updateMlHyperParameters(MlModel origModel, MlModel updatedModel) throws JsonProcessingException {
recordChange("mlHyperParameters", origModel.getMlHyperParameters(), updatedModel.getMlHyperParameters());
recordChange("mlHyperParameters", origModel.getMlHyperParameters(), updatedModel.getMlHyperParameters(), true);
}
private void updateMlStore(MlModel origModel, MlModel updatedModel) throws JsonProcessingException {
@ -358,7 +362,11 @@ public class MlModelRepository extends EntityRepository<MlModel> {
}
private void updateServer(MlModel origModel, MlModel updatedModel) throws JsonProcessingException {
recordChange("server", origModel.getServer(), updatedModel.getServer());
// Updating the server can break current integrations to the ML services or enable new integrations
if (recordChange("server", origModel.getServer(), updatedModel.getServer())) {
// Mark the EntityUpdater version change to major
majorVersionChange = true;
}
}
private void updateDashboard(MlModel origModel, MlModel updatedModel) throws JsonProcessingException {

73
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/mlmodels/MlModelResourceTest.java
View File

@ -69,6 +69,7 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.ENTITY_ALREADY_EXISTS;
import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityNotFound;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MAJOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.MINOR_UPDATE;
import static org.openmetadata.catalog.util.TestUtils.UpdateType.NO_CHANGE;
import static org.openmetadata.catalog.util.TestUtils.adminAuthHeaders;
@ -243,7 +244,7 @@ public class MlModelResourceTest extends EntityResourceTest<MlModel> {
new FieldChange().withName("algorithm").withNewValue("SVM").withOldValue("regression")
);
updateAndCheckEntity(request.withAlgorithm("SVM"), Status.OK, adminAuthHeaders(), MINOR_UPDATE, change);
updateAndCheckEntity(request.withAlgorithm("SVM"), Status.OK, adminAuthHeaders(), MAJOR_UPDATE, change);
}
@Test
@ -266,7 +267,23 @@ public class MlModelResourceTest extends EntityResourceTest<MlModel> {
change.getFieldsAdded().add(new FieldChange().withName("server").withNewValue(SERVER));
updateAndCheckEntity(
request.withServer(SERVER), Status.OK, adminAuthHeaders(), MINOR_UPDATE, change
request.withServer(SERVER), Status.OK, adminAuthHeaders(), MAJOR_UPDATE, change
);
}
@Test
public void put_MlModelUpdateServer_200(TestInfo test) throws IOException {
CreateMlModel request = create(test).withServer(SERVER);
MlModel model = createAndCheckEntity(request, adminAuthHeaders());
ChangeDescription change = getChangeDescription(model.getVersion());
URI newServer = URI.create("http://localhost.com/mlModel/v2");
change.getFieldsUpdated().add(
new FieldChange().withName("server").withNewValue(newServer).withOldValue(SERVER)
);
updateAndCheckEntity(
request.withServer(newServer), Status.OK, adminAuthHeaders(), MAJOR_UPDATE, change
);
}
@ -282,6 +299,51 @@ public class MlModelResourceTest extends EntityResourceTest<MlModel> {
);
}
@Test
public void put_MlModelAddMlFeatures_200(TestInfo test) throws IOException {
CreateMlModel request = new CreateMlModel().withName(getEntityName(test)).withAlgorithm(ALGORITHM);
MlModel model = createAndCheckEntity(request, adminAuthHeaders());
ChangeDescription change = getChangeDescription(model.getVersion());
change.getFieldsAdded().add(new FieldChange().withName("mlFeatures").withNewValue(ML_FEATURES));
updateAndCheckEntity(
request.withMlFeatures(ML_FEATURES), Status.OK, adminAuthHeaders(), MINOR_UPDATE, change
);
}
@Test
public void put_MlModelUpdateMlFeatures_200(TestInfo test) throws IOException {
CreateMlModel request = create(test);
MlModel model = createAndCheckEntity(request, adminAuthHeaders());
ChangeDescription change = getChangeDescription(model.getVersion());
MlFeature newMlFeature = new MlFeature()
.withName("color")
.withDataType(MlFeatureDataType.Categorical);
List<MlFeature> newFeatures = Collections.singletonList(newMlFeature);
change.getFieldsUpdated().add(
new FieldChange().withName("mlFeatures").withNewValue(newFeatures).withOldValue(ML_FEATURES)
);
updateAndCheckEntity(
request.withMlFeatures(newFeatures), Status.OK, adminAuthHeaders(), MINOR_UPDATE, change
);
}
@Test
public void put_MlModelAddMlHyperParams_200(TestInfo test) throws IOException {
CreateMlModel request = new CreateMlModel().withName(getEntityName(test)).withAlgorithm(ALGORITHM);
MlModel model = createAndCheckEntity(request, adminAuthHeaders());
ChangeDescription change = getChangeDescription(model.getVersion());
change.getFieldsAdded().add(new FieldChange().withName("mlHyperParameters").withNewValue(ML_HYPERPARAMS));
updateAndCheckEntity(
request.withMlHyperParameters(ML_HYPERPARAMS), Status.OK, adminAuthHeaders(), MINOR_UPDATE, change
);
}
@Test
public void get_nonExistentMlModel_404_notFound() {
HttpResponseException exception = assertThrows(HttpResponseException.class, () ->
@ -293,7 +355,6 @@ public class MlModelResourceTest extends EntityResourceTest<MlModel> {
@Test
public void get_MlModelWithDifferentFields_200_OK(TestInfo test) throws IOException {
// aqui no tenim HREF al dashboard
CreateMlModel create = create(test).withDescription("description")
.withOwner(USER_OWNER1).withDashboard(DASHBOARD_REFERENCE);
MlModel model = createAndCheckEntity(create, adminAuthHeaders());
@ -431,6 +492,7 @@ public class MlModelResourceTest extends EntityResourceTest<MlModel> {
}
BiConsumer<MlFeature, MlFeature> assertMlFeature = (MlFeature expected, MlFeature actual) -> {
// FQN gets created on-the-fly based on the test name. Just check that it is not null
assertNotNull(actual.getFullyQualifiedName());
assertEquals(actual.getName(), expected.getName());
assertEquals(actual.getDescription(), expected.getDescription());
@ -447,6 +509,7 @@ public class MlModelResourceTest extends EntityResourceTest<MlModel> {
BiConsumer<MlFeatureSource, MlFeatureSource> assertMlFeatureSource =
(MlFeatureSource expected, MlFeatureSource actual) -> {
// FQN gets created on-the-fly based on the test name. Just check that it is not null
assertNotNull(actual.getFullyQualifiedName());
assertEquals(actual.getName(), expected.getName());
assertEquals(actual.getDescription(), expected.getDescription());
@ -492,11 +555,11 @@ public class MlModelResourceTest extends EntityResourceTest<MlModel> {
if (fieldName.contains("mlFeatures")) {
List<MlFeature> expectedFeatures = (List<MlFeature>) expected;
List<MlFeature> actualFeatures = JsonUtils.readObjects(actual.toString(), MlFeature.class);
assertEquals(expectedFeatures, actualFeatures);
assertListProperty(expectedFeatures, actualFeatures, assertMlFeature);
} else if (fieldName.contains("mlHyperParameters")) {
List<MlHyperParameter> expectedConstraints = (List<MlHyperParameter>) expected;
List<MlHyperParameter> actualConstraints = JsonUtils.readObjects(actual.toString(), MlHyperParameter.class);
assertEquals(expectedConstraints, actualConstraints);
assertListProperty(expectedConstraints, actualConstraints, assertMlHyperParam);
} else if (fieldName.contains("algorithm")) {
String expectedAlgorithm = (String) expected;
String actualAlgorithm = actual.toString();