diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java index 95ae309d00c..2285d608a5f 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/CatalogApplication.java @@ -211,7 +211,7 @@ public class CatalogApplication extends Application { "There are pending migrations to be run on the database." + " Please backup your data and run `./bootstrap/bootstrap_storage.sh migrate-all`." + " You can find more information on upgrading OpenMetadata at" - + " https://docs.open-metadata.org/install/upgrade-openmetadata"); + + " https://docs.open-metadata.org/deployment/upgrade"); } } diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/airflow/AirflowRESTClient.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/airflow/AirflowRESTClient.java index b56647486e8..4887d4966fc 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/airflow/AirflowRESTClient.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/airflow/AirflowRESTClient.java @@ -230,6 +230,21 @@ public class AirflowRESTClient extends PipelineServiceClient { String.format("Failed to kill running workflows due to %s", response.body())); } + @Override + public Map getHostIp() { + HttpResponse response; + try { + response = getRequestAuthenticatedForJsonContent("%s/%s/ip", serviceURL, API_ENDPOINT); + if (response.statusCode() == 200) { + return JsonUtils.readValue(response.body(), new TypeReference<>() {}); + } + } catch (Exception e) { + throw PipelineServiceClientException.byMessage("Failed to get Pipeline Service host IP.", e.getMessage()); + } + throw new PipelineServiceClientException( + String.format("Failed to get Pipeline Service host IP due to %s", response.body())); + } + @Override public Map getLastIngestionLogs(IngestionPipeline ingestionPipeline) { HttpResponse response; diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java index 628098390af..fe41f40b8bf 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/resources/services/ingestionpipelines/IngestionPipelineResource.java @@ -520,6 +520,24 @@ public class IngestionPipelineResource extends EntityResource hostIp = pipelineServiceClient.getHostIp(); + return Response.ok(hostIp, MediaType.APPLICATION_JSON_TYPE).build(); + } + @DELETE @Path("/{id}") @Operation( diff --git a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/PipelineServiceClient.java b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/PipelineServiceClient.java index 91e3d7f81b5..937b6cc4315 100644 --- a/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/PipelineServiceClient.java +++ b/catalog-rest-service/src/main/java/org/openmetadata/catalog/util/PipelineServiceClient.java @@ -141,4 +141,7 @@ public abstract class PipelineServiceClient { /* Get the all last run logs of a deployed pipeline */ public abstract HttpResponse killIngestion(IngestionPipeline ingestionPipeline); + + /* Get the Pipeline Service host IP to whitelist in source systems */ + public abstract Map getHostIp(); } diff --git a/catalog-rest-service/src/test/java/org/openmetadata/catalog/pipelineService/MockPipelineServiceClient.java b/catalog-rest-service/src/test/java/org/openmetadata/catalog/pipelineService/MockPipelineServiceClient.java index 1f06a33be10..c6df5682b2a 100644 --- a/catalog-rest-service/src/test/java/org/openmetadata/catalog/pipelineService/MockPipelineServiceClient.java +++ b/catalog-rest-service/src/test/java/org/openmetadata/catalog/pipelineService/MockPipelineServiceClient.java @@ -57,4 +57,9 @@ public class MockPipelineServiceClient extends PipelineServiceClient { public HttpResponse killIngestion(IngestionPipeline ingestionPipeline) { return null; } + + @Override + public Map getHostIp() { + return null; + } } diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/api/routes/ip.py b/openmetadata-airflow-apis/openmetadata_managed_apis/api/routes/ip.py new file mode 100644 index 00000000000..1fce7ba5012 --- /dev/null +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/api/routes/ip.py @@ -0,0 +1,45 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +IP endpoint +""" +import traceback + +import requests + +try: + from importlib.metadata import version +except ImportError: + from importlib_metadata import version + +from airflow.api_connexion import security +from airflow.security import permissions +from airflow.www.app import csrf +from openmetadata_managed_apis.api.app import blueprint +from openmetadata_managed_apis.api.response import ApiResponse + + +@blueprint.route("/ip", methods=["GET"]) +@csrf.exempt +@security.requires_access([(permissions.ACTION_CAN_EDIT, permissions.RESOURCE_DAG)]) +def get_host_ip(): + """ + /ip endpoint to check Airflow host IP. Users will need to whitelist + this IP to access their source systems. + """ + + try: + return ApiResponse.success({"ip": requests.get("https://api.ipify.org").text}) + except Exception as err: + return ApiResponse.error( + status=ApiResponse.STATUS_SERVER_ERROR, + error=f"Internal error obtaining host IP - {err} - {traceback.format_exc()}", + ) diff --git a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/ingestionPipelineAPI.ts b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/ingestionPipelineAPI.ts index 1bdc0194f37..f7313d93f2d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/ingestionPipelineAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/axiosAPIs/ingestionPipelineAPI.ts @@ -100,6 +100,14 @@ export const checkAirflowStatus = (): Promise => { return APIClient.get('/services/ingestionPipelines/status'); }; +export const getPipelineServiceHostIp = async () => { + const response = await APIClient.get<{ ip?: string }>( + '/services/ingestionPipelines/ip' + ); + + return response.data; +}; + export const getIngestionPipelineLogById = ( id: string ): Promise => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx index 8d4549543eb..d502df4a8f6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx @@ -13,13 +13,16 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import Form, { FormProps } from '@rjsf/core'; +import { AxiosError } from 'axios'; import classNames from 'classnames'; import { isEmpty } from 'lodash'; import { LoadingState } from 'Models'; -import React, { FunctionComponent, useState } from 'react'; +import React, { FunctionComponent, useEffect, useState } from 'react'; +import { getPipelineServiceHostIp } from '../../../axiosAPIs/ingestionPipelineAPI'; import { ConfigData } from '../../../interface/service.interface'; import { formatFormDataForRender } from '../../../utils/JSONSchemaFormUtils'; import SVGIcons, { Icons } from '../../../utils/SvgUtils'; +import { showErrorToast } from '../../../utils/ToastUtils'; import { Button } from '../../buttons/Button/Button'; import { ArrayFieldTemplate } from '../../JSONSchemaTemplate/ArrayFieldTemplate'; import { ObjectFieldTemplate } from '../../JSONSchemaTemplate/ObjectFieldTemplate'; @@ -55,6 +58,22 @@ const FormBuilder: FunctionComponent = ({ const [connectionTestingState, setConnectionTestingState] = useState('initial'); + const [hostIp, setHostIp] = useState('[fetching]'); + + const fetchHostIp = async () => { + try { + const data = await getPipelineServiceHostIp(); + setHostIp(data?.ip || '[unknown]'); + } catch (error) { + setHostIp('[error - unknown]'); + showErrorToast(error as AxiosError); + } + }; + + useEffect(() => { + fetchHostIp(); + }, []); + const handleCancel = () => { setLocalFormData(formatFormDataForRender(formData)); if (onCancel) { @@ -108,10 +127,22 @@ const FormBuilder: FunctionComponent = ({ case 'initial': default: - return 'Test your connections before creating service'; + return 'Test your connections before creating the service'; } }; + const getPipelineServiceHostIpForm = () => { + return ( +
+ + + OpenMetadata will connect to your resource from the IP {hostIp}. Make + sure to allow inbound traffic in your network security settings. + +
+ ); + }; + return (
= ({ No Connection Configs available. )} + {!isEmpty(schema) && onTestConnection && ( +
+
{getPipelineServiceHostIpForm()}
+
+ )} {!isEmpty(schema) && onTestConnection && (
{getConnectionTestingMessage()}