yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-09 09:38:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #1202: IngestionResource put/patch not updating entities (#1203)

Browse Source 
* Fix #1202: IngestionResource put/patch not updating entities

* fix for failing tests

* addressing reviewdog finding

Co-authored-by: Parth Panchal <parth.panchal@deuexsolutions.com>
This commit is contained in:
Sriharsha Chintalapani 2021-11-16 04:14:44 -08:00 committed by GitHub
parent c760ebf281
commit 38761ea414
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 19 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionRepository.java
View File

@ -41,9 +41,9 @@ import static org.openmetadata.catalog.exception.CatalogExceptionMessage.entityN
public class IngestionRepository extends EntityRepository<Ingestion> { public class IngestionRepository extends EntityRepository<Ingestion> {
private static final Fields INGESTION_UPDATE_FIELDS = new Fields(IngestionResource.FIELD_LIST, private static final Fields INGESTION_UPDATE_FIELDS = new Fields(IngestionResource.FIELD_LIST,
"owner,tags"); "scheduleInterval,owner,tags");
private static final Fields INGESTION_PATCH_FIELDS = new Fields(IngestionResource.FIELD_LIST, private static final Fields INGESTION_PATCH_FIELDS = new Fields(IngestionResource.FIELD_LIST,
"owner,tags"); "scheduleInterval,owner,tags");
private final CollectionDAO dao; private final CollectionDAO dao;
public IngestionRepository(CollectionDAO dao) { public IngestionRepository(CollectionDAO dao) {
@ -76,6 +76,8 @@ public class IngestionRepository extends EntityRepository<Ingestion> {
public Ingestion setFields(Ingestion ingestion, Fields fields) throws IOException { public Ingestion setFields(Ingestion ingestion, Fields fields) throws IOException {
ingestion.setDisplayName(ingestion.getDisplayName()); ingestion.setDisplayName(ingestion.getDisplayName());
ingestion.setService(getService(ingestion)); ingestion.setService(getService(ingestion));
ingestion.setConnectorConfig(ingestion.getConnectorConfig());
ingestion.setScheduleInterval(ingestion.getScheduleInterval());
ingestion.setOwner(fields.contains("owner") ? getOwner(ingestion) : null); ingestion.setOwner(fields.contains("owner") ? getOwner(ingestion) : null);
ingestion.setTags(fields.contains("tags") ? getTags(ingestion.getFullyQualifiedName()) : null); ingestion.setTags(fields.contains("tags") ? getTags(ingestion.getFullyQualifiedName()) : null);
return ingestion; return ingestion;
@ -285,7 +287,12 @@ public class IngestionRepository extends EntityRepository<Ingestion> {
@Override @Override
public void entitySpecificUpdate() throws IOException { public void entitySpecificUpdate() throws IOException {
Ingestion origIngestion = original.getEntity();
Ingestion updatedIngestion = updated.getEntity();
recordChange("scheduleInterval", origIngestion.getScheduleInterval(), updatedIngestion.getScheduleInterval());
recordChange("connectorConfig", origIngestion.getConnectorConfig(), updatedIngestion.getConnectorConfig());
recordChange("startDate", origIngestion.getStartDate(), updatedIngestion.getStartDate());
recordChange("endDate", origIngestion.getEndDate(), updatedIngestion.getEndDate());
} }
} }

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/operations/IngestionResource.java
View File

@ -131,7 +131,7 @@ public class IngestionResource {
} }
} }
static final String FIELDS = "owner,service,tags,status"; static final String FIELDS = "owner,service,tags,status,scheduleInterval";
public static final List<String> FIELD_LIST = Arrays.asList(FIELDS.replaceAll(" ", "") public static final List<String> FIELD_LIST = Arrays.asList(FIELDS.replaceAll(" ", "")
.split(",")); .split(","));

8
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/operations/IngestionResourceTest.java
View File

@ -221,19 +221,23 @@ public class IngestionResourceTest extends EntityResourceTest<Ingestion> {
@Test @Test
public void put_IngestionUrlUpdate_200(TestInfo test) throws IOException { public void put_IngestionUrlUpdate_200(TestInfo test) throws IOException {
CreateIngestion request = create(test).withService(new EntityReference().withId(BIGQUERY_REFERENCE.getId()) CreateIngestion request = create(test).withService(new EntityReference().withId(BIGQUERY_REFERENCE.getId())
.withType("databaseService")).withDescription("description"); .withType("databaseService")).withDescription("description").withScheduleInterval("5 * * * *");
createAndCheckEntity(request, adminAuthHeaders()); createAndCheckEntity(request, adminAuthHeaders());
Integer pipelineConcurrency = 110; Integer pipelineConcurrency = 110;
Date startDate = new DateTime("2021-11-13T20:20:39+00:00").toDate(); Date startDate = new DateTime("2021-11-13T20:20:39+00:00").toDate();
String expectedScheduleInterval = "7 * * * *";
// Updating description is ignored when backend already has description // Updating description is ignored when backend already has description
Ingestion ingestion = updateIngestion(request.withConnectorConfig(INGESTION_CONFIG) Ingestion ingestion = updateIngestion(request.withConnectorConfig(INGESTION_CONFIG)
.withConcurrency(pipelineConcurrency) .withConcurrency(pipelineConcurrency)
.withScheduleInterval(expectedScheduleInterval)
.withStartDate(startDate.toString()), OK, adminAuthHeaders()); .withStartDate(startDate.toString()), OK, adminAuthHeaders());
String expectedFQN = BIGQUERY_REFERENCE.getName() + "." + ingestion.getName(); String expectedFQN = BIGQUERY_REFERENCE.getName() + "." + ingestion.getName();
assertEquals(startDate.toString(), ingestion.getStartDate()); assertEquals(startDate.toString(), ingestion.getStartDate());
assertEquals(pipelineConcurrency, ingestion.getConcurrency()); assertEquals(pipelineConcurrency, ingestion.getConcurrency());
assertEquals(expectedFQN, ingestion.getFullyQualifiedName()); assertEquals(expectedFQN, ingestion.getFullyQualifiedName());
assertEquals(expectedScheduleInterval, ingestion.getScheduleInterval());
ingestion = getIngestion(ingestion.getId(), "owner,service", adminAuthHeaders());
assertEquals(expectedScheduleInterval, ingestion.getScheduleInterval());
} }
@Test @Test