yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-08 21:46:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix #4509: Ingestion Pipeline changes are not getting stored (#4510)

Browse Source
This commit is contained in:
Sriharsha Chintalapani 2022-04-27 00:00:19 -07:00 committed by GitHub
parent 1986423f9d
commit 55c91b2a27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 14 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
catalog-rest-service/src/main/java/org/openmetadata/catalog/airflow/AirflowRESTClient.java
View File

@ -207,10 +207,9 @@ public class AirflowRESTClient {
if (response.statusCode() == 200) {
return response;
}
} catch (Exception e) {
throw AirflowException.byMessage("Failed to test connection.", e.getMessage());
}
throw AirflowException.byMessage("Failed to test connection.", response.toString());
throw new AirflowException(String.format("Failed to test connection due to %s", response.body()));
}
}

2
catalog-rest-service/src/main/java/org/openmetadata/catalog/elasticsearch/ElasticSearchEventPublisher.java
View File

@ -139,7 +139,7 @@ public class ElasticSearchEventPublisher extends AbstractEventPublisher {
LOG.error("Error in publishing to ElasticSearch");
throw new ElasticSearchRetriableException(e.getMessage());
} else {
throw new EventPublisherException(e.getMessage());
LOG.error(e.getMessage(), e);
}
} catch (IOException ie) {
throw new EventPublisherException(ie.getMessage());

14
catalog-rest-service/src/main/java/org/openmetadata/catalog/jdbi3/IngestionPipelineRepository.java
View File

@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
import java.net.URI;
import java.util.UUID;
import org.json.JSONObject;
import org.openmetadata.catalog.Entity;
import org.openmetadata.catalog.entity.services.ingestionPipelines.AirflowConfig;
import org.openmetadata.catalog.entity.services.ingestionPipelines.IngestionPipeline;
@ -31,6 +32,7 @@ import org.openmetadata.catalog.type.Relationship;
import org.openmetadata.catalog.util.EntityInterface;
import org.openmetadata.catalog.util.EntityUtil.Fields;
import org.openmetadata.catalog.util.FullyQualifiedName;
import org.openmetadata.catalog.util.JsonUtils;
public class IngestionPipelineRepository extends EntityRepository<IngestionPipeline> {
private static final String INGESTION_PIPELINE_UPDATE_FIELDS = "owner,source,airflowConfig";
@ -253,9 +255,15 @@ public class IngestionPipelineRepository extends EntityRepository<IngestionPipel
}
private void updateSource(Source origSource, Source updatedSource) throws JsonProcessingException {
if (origSource.getServiceConnection() != updatedSource.getServiceConnection()
&& !origSource.getServiceName().equals(updatedSource.getServiceName())
&& origSource.getSourceConfig() != updatedSource.getSourceConfig()) {
JSONObject origSourceConfig = new JSONObject(JsonUtils.pojoToJson(origSource.getSourceConfig().getConfig()));
JSONObject updatedSourceConfig =
new JSONObject(JsonUtils.pojoToJson(updatedSource.getSourceConfig().getConfig()));
JSONObject origSourceConnection = new JSONObject(JsonUtils.pojoToJson(origSource.getServiceConnection()));
JSONObject updatedSourceConnection = new JSONObject(JsonUtils.pojoToJson(updatedSource.getServiceConnection()));
if (!origSource.getServiceName().equals(updatedSource.getServiceName())
|| !origSourceConfig.similar(updatedSourceConfig)
|| !origSourceConnection.similar(updatedSourceConnection)) {
recordChange("source", origSource, updatedSource);
}
}

1
catalog-rest-service/src/test/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResourceTest.java
View File

@ -89,6 +89,7 @@ public class IngestionPipelineResourceTest extends EntityResourceTest<IngestionP
IngestionPipelineResource.IngestionPipelineList.class,
"services/ingestionPipelines",
IngestionPipelineResource.FIELDS);
this.supportsEmptyDescription = false;
}
@BeforeAll