Add ingestionIpInfoEnabled configuration (#10870)

* Add ingestionIpInfoEnabled configuration

* Format

* hide ip address if api response if not 200

* removing dependancy on ip-modal and checking default state

* updated status code for ip to 204 to check default state

* fixed failing cypress

---------

Co-authored-by: Chirag Madlani <12962843+chirag-madlani@users.noreply.github.com>
Co-authored-by: Shailesh Parmar <shailesh.parmar.webdev@gmail.com>

conf/openmetadata.yaml

@@ -244,6 +244,7 @@ pipelineServiceClientConfiguration:
   className: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
   apiEndpoint: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://localhost:8080}
   metadataApiEndpoint: ${SERVER_HOST_API_URL:-http://localhost:8585/api}
+  ingestionIpInfoEnabled: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
   hostIp: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
   verifySSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"} # Possible values are "no-ssl", "ignore", "validate"
   sslConfig:

openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java

@@ -526,8 +526,7 @@ public class IngestionPipelineResource extends EntityResource<IngestionPipeline,
             content = @Content(mediaType = "application/json"))
       })
   public Response getHostIp(@Context UriInfo uriInfo, @Context SecurityContext securityContext) {
-    Map<String, String> hostIp = pipelineServiceClient.getHostIp();
-    return Response.ok(hostIp, MediaType.APPLICATION_JSON_TYPE).build();
+    return pipelineServiceClient.getHostIp();
   }
 
   @GET

openmetadata-spec/src/main/java/org/openmetadata/sdk/PipelineServiceClient.java

@@ -22,6 +22,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.regex.Pattern;
+import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.common.utils.CommonUtil;
@@ -50,6 +51,8 @@ import org.openmetadata.sdk.exception.PipelineServiceVersionException;
 public abstract class PipelineServiceClient {
   protected final String hostIp;
 
+  protected final boolean ingestionIpInfoEnabled;
+
   protected static final String AUTH_HEADER = "Authorization";
   protected static final String CONTENT_HEADER = "Content-Type";
   protected static final String CONTENT_TYPE = "application/json";
@@ -87,6 +90,7 @@ public abstract class PipelineServiceClient {
 
   public PipelineServiceClient(PipelineServiceClientConfiguration pipelineServiceClientConfiguration) {
     this.hostIp = pipelineServiceClientConfiguration.getHostIp();
+    this.ingestionIpInfoEnabled = pipelineServiceClientConfiguration.getIngestionIpInfoEnabled();
   }
 
   public final URL validateServiceURL(String serviceURL) {
@@ -128,15 +132,28 @@ public abstract class PipelineServiceClient {
     return getVersionFromString(clientVersion).equals(getVersionFromString(SERVER_VERSION));
   }
 
-  public final Map<String, String> getHostIp() {
+  public final Response getHostIp() {
+
+    if (this.ingestionIpInfoEnabled) {
+      return getHostIpInternal();
+    }
+    return Response.status(Response.Status.NO_CONTENT).build();
+  }
+
+  private Response getHostIpInternal() {
+    Map<String, String> body;
     try {
-      return CommonUtil.nullOrEmpty(this.hostIp) ? requestGetHostIp() : Map.of("ip", this.hostIp);
+      body = CommonUtil.nullOrEmpty(this.hostIp) ? requestGetHostIp() : Map.of("ip", this.hostIp);
+      return Response.ok(body, MediaType.APPLICATION_JSON_TYPE).build();
     } catch (Exception e) {
       LOG.error("Failed to get Pipeline Service host IP. {}", e.getMessage());
-      return Map.of(
-          "ip",
-          "Failed to find the IP of Airflow Container. Please make sure https://api.ipify.org, "
-              + "https://api.my-ip.io/ip reachable from your network or that the `hostIp` setting is configured.");
+      // We don't want the request to fail for an informative ping
+      body =
+          Map.of(
+              "ip",
+              "Failed to find the IP of Airflow Container. Please make sure https://api.ipify.org, "
+                  + "https://api.my-ip.io/ip reachable from your network or that the `hostIp` setting is configured.");
+      return Response.ok(body, MediaType.APPLICATION_JSON_TYPE).build();
     }
   }
 

openmetadata-spec/src/main/resources/json/schema/configuration/pipelineServiceClientConfiguration.json

@@ -18,6 +18,11 @@
       "description": "Pipeline Service Client host IP that will be used to connect to the sources.",
       "type": "string"
     },
+    "ingestionIpInfoEnabled": {
+      "description": "Enable or disable the API that fetches the public IP running the ingestion process.",
+      "type": "boolean",
+      "default": false
+    },
     "metadataApiEndpoint": {
       "description": "Metadata api endpoint, e.g., `http://localhost:8585/api`",
       "type": "string"

openmetadata-ui/src/main/resources/ui/cypress/common/common.js

@@ -181,13 +181,15 @@ export const testServiceCreationAndIngestion = (
 
   // Enter service name in step 3
   cy.get('[data-testid="service-name"]').should('exist').type(serviceName);
+  interceptURL('GET', '/api/v1/services/ingestionPipelines/ip', 'ipApi');
   interceptURL(
     'GET',
     'api/v1/services/ingestionPipelines/*',
-    'getIngestionPipelineStatus'
+    'ingestionPipelineStatus'
   );
   cy.get('[data-testid="next-button"]').should('exist').click();
-  verifyResponseStatusCode('@getIngestionPipelineStatus', 200);
+  verifyResponseStatusCode('@ingestionPipelineStatus', 200);
+  verifyResponseStatusCode('@ipApi', 204);
 
   // Connection Details in step 4
   cy.get('[data-testid="add-new-service-container"]')
@@ -200,9 +202,6 @@ export const testServiceCreationAndIngestion = (
 
   connectionInput();
 
-  // check for the ip-address widget
-  cy.get('[data-testid="ip-address"]').should('exist');
-
   // Test the connection
   interceptURL(
     'GET',

openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx

@@ -67,12 +67,16 @@ const FormBuilder: FunctionComponent<Props> = ({
     formatFormDataForRender(formData ?? {})
   );
 
-  const [hostIp, setHostIp] = useState<string>('[fetching]');
+  const [hostIp, setHostIp] = useState<string>();
 
   const fetchHostIp = async () => {
     try {
-      const data = await getPipelineServiceHostIp();
-      setHostIp(data?.ip || '[unknown]');
+      const { status, data } = await getPipelineServiceHostIp();
+      if (status === 200) {
+        setHostIp(data?.ip || '[unknown]');
+      } else {
+        setHostIp(undefined);
+      }
     } catch (error) {
       setHostIp('[error - unknown]');
     }
@@ -136,7 +140,7 @@ const FormBuilder: FunctionComponent<Props> = ({
           {t('message.no-config-available')}
         </div>
       )}
-      {!isEmpty(schema) && isAirflowAvailable && (
+      {!isEmpty(schema) && isAirflowAvailable && hostIp && (
         <div
           className="tw-flex tw-justify-between tw-bg-white tw-border tw-border-main tw-shadow tw-rounded tw-p-3 tw-mt-4"
           data-testid="ip-address">

openmetadata-ui/src/main/resources/ui/src/rest/ingestionPipelineAPI.ts

@@ -153,7 +153,7 @@ export const getPipelineServiceHostIp = async () => {
     '/services/ingestionPipelines/ip'
   );
 
-  return response.data;
+  return response;
 };
 
 export const getIngestionPipelineLogById = (id: string, after?: string) => {