diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml index 7b6bb9804cb..856753535a9 100644 --- a/conf/openmetadata.yaml +++ b/conf/openmetadata.yaml @@ -244,6 +244,7 @@ pipelineServiceClientConfiguration: className: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"} apiEndpoint: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://localhost:8080} metadataApiEndpoint: ${SERVER_HOST_API_URL:-http://localhost:8585/api} + ingestionIpInfoEnabled: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false} hostIp: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""} verifySSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"} # Possible values are "no-ssl", "ignore", "validate" sslConfig: diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java index a737eaa44d8..78a70cad8d3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java @@ -526,8 +526,7 @@ public class IngestionPipelineResource extends EntityResource hostIp = pipelineServiceClient.getHostIp(); - return Response.ok(hostIp, MediaType.APPLICATION_JSON_TYPE).build(); + return pipelineServiceClient.getHostIp(); } @GET diff --git a/openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClient.java b/openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClient.java index f512a158d23..513f924d6d9 100644 --- a/openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClient.java +++ b/openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClient.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.regex.Pattern; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; import org.openmetadata.common.utils.CommonUtil; @@ -50,6 +51,8 @@ import org.openmetadata.sdk.exception.PipelineServiceVersionException; public abstract class PipelineServiceClient { protected final String hostIp; + protected final boolean ingestionIpInfoEnabled; + protected static final String AUTH_HEADER = "Authorization"; protected static final String CONTENT_HEADER = "Content-Type"; protected static final String CONTENT_TYPE = "application/json"; @@ -87,6 +90,7 @@ public abstract class PipelineServiceClient { public PipelineServiceClient(PipelineServiceClientConfiguration pipelineServiceClientConfiguration) { this.hostIp = pipelineServiceClientConfiguration.getHostIp(); + this.ingestionIpInfoEnabled = pipelineServiceClientConfiguration.getIngestionIpInfoEnabled(); } public final URL validateServiceURL(String serviceURL) { @@ -128,15 +132,28 @@ public abstract class PipelineServiceClient { return getVersionFromString(clientVersion).equals(getVersionFromString(SERVER_VERSION)); } - public final Map getHostIp() { + public final Response getHostIp() { + + if (this.ingestionIpInfoEnabled) { + return getHostIpInternal(); + } + return Response.status(Response.Status.NO_CONTENT).build(); + } + + private Response getHostIpInternal() { + Map body; try { - return CommonUtil.nullOrEmpty(this.hostIp) ? requestGetHostIp() : Map.of("ip", this.hostIp); + body = CommonUtil.nullOrEmpty(this.hostIp) ? requestGetHostIp() : Map.of("ip", this.hostIp); + return Response.ok(body, MediaType.APPLICATION_JSON_TYPE).build(); } catch (Exception e) { LOG.error("Failed to get Pipeline Service host IP. {}", e.getMessage()); - return Map.of( - "ip", - "Failed to find the IP of Airflow Container. Please make sure https://api.ipify.org, " - + "https://api.my-ip.io/ip reachable from your network or that the `hostIp` setting is configured."); + // We don't want the request to fail for an informative ping + body = + Map.of( + "ip", + "Failed to find the IP of Airflow Container. Please make sure https://api.ipify.org, " + + "https://api.my-ip.io/ip reachable from your network or that the `hostIp` setting is configured."); + return Response.ok(body, MediaType.APPLICATION_JSON_TYPE).build(); } } diff --git a/openmetadata-spec/src/main/resources/json/schema/configuration/pipelineServiceClientConfiguration.json b/openmetadata-spec/src/main/resources/json/schema/configuration/pipelineServiceClientConfiguration.json index 0945d70e8e3..cf462d7bb0b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/configuration/pipelineServiceClientConfiguration.json +++ b/openmetadata-spec/src/main/resources/json/schema/configuration/pipelineServiceClientConfiguration.json @@ -18,6 +18,11 @@ "description": "Pipeline Service Client host IP that will be used to connect to the sources.", "type": "string" }, + "ingestionIpInfoEnabled": { + "description": "Enable or disable the API that fetches the public IP running the ingestion process.", + "type": "boolean", + "default": false + }, "metadataApiEndpoint": { "description": "Metadata api endpoint, e.g., `http://localhost:8585/api`", "type": "string" diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js index 3951758d41f..cf33ad98fff 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/common.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/common.js @@ -181,13 +181,15 @@ export const testServiceCreationAndIngestion = ( // Enter service name in step 3 cy.get('[data-testid="service-name"]').should('exist').type(serviceName); + interceptURL('GET', '/api/v1/services/ingestionPipelines/ip', 'ipApi'); interceptURL( 'GET', 'api/v1/services/ingestionPipelines/*', - 'getIngestionPipelineStatus' + 'ingestionPipelineStatus' ); cy.get('[data-testid="next-button"]').should('exist').click(); - verifyResponseStatusCode('@getIngestionPipelineStatus', 200); + verifyResponseStatusCode('@ingestionPipelineStatus', 200); + verifyResponseStatusCode('@ipApi', 204); // Connection Details in step 4 cy.get('[data-testid="add-new-service-container"]') @@ -200,9 +202,6 @@ export const testServiceCreationAndIngestion = ( connectionInput(); - // check for the ip-address widget - cy.get('[data-testid="ip-address"]').should('exist'); - // Test the connection interceptURL( 'GET', diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx index 0c0a5ce3357..86a931d7e25 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx @@ -67,12 +67,16 @@ const FormBuilder: FunctionComponent = ({ formatFormDataForRender(formData ?? {}) ); - const [hostIp, setHostIp] = useState('[fetching]'); + const [hostIp, setHostIp] = useState(); const fetchHostIp = async () => { try { - const data = await getPipelineServiceHostIp(); - setHostIp(data?.ip || '[unknown]'); + const { status, data } = await getPipelineServiceHostIp(); + if (status === 200) { + setHostIp(data?.ip || '[unknown]'); + } else { + setHostIp(undefined); + } } catch (error) { setHostIp('[error - unknown]'); } @@ -136,7 +140,7 @@ const FormBuilder: FunctionComponent = ({ {t('message.no-config-available')} )} - {!isEmpty(schema) && isAirflowAvailable && ( + {!isEmpty(schema) && isAirflowAvailable && hostIp && (
diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/ingestionPipelineAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/ingestionPipelineAPI.ts index f4ff257fd18..72691a5a15f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/ingestionPipelineAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/ingestionPipelineAPI.ts @@ -153,7 +153,7 @@ export const getPipelineServiceHostIp = async () => { '/services/ingestionPipelines/ip' ); - return response.data; + return response; }; export const getIngestionPipelineLogById = (id: string, after?: string) => {