From cf08637d5fac6801f5dca8fc800d5579d8ec7308 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Sat, 22 Oct 2022 12:31:32 +0200 Subject: [PATCH] Airflow APIs docs (#8312) --- .../openmetadata_managed_apis/__init__.py | 2 +- .../views/templates/rest_api/index.html | 2 +- .../content/deployment/airflow/index.md | 49 ++++++++++++++++--- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/__init__.py b/openmetadata-airflow-apis/openmetadata_managed_apis/__init__.py index e5c0af8a138..7978359fb0c 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/__init__.py +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/__init__.py @@ -9,4 +9,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.11" +__version__ = "0.12.2.0" diff --git a/openmetadata-airflow-apis/openmetadata_managed_apis/views/templates/rest_api/index.html b/openmetadata-airflow-apis/openmetadata_managed_apis/views/templates/rest_api/index.html index 12b11aa1da5..dfc4bbf88b8 100644 --- a/openmetadata-airflow-apis/openmetadata_managed_apis/views/templates/rest_api/index.html +++ b/openmetadata-airflow-apis/openmetadata_managed_apis/views/templates/rest_api/index.html @@ -73,7 +73,7 @@

Documentation

diff --git a/openmetadata-docs/content/deployment/airflow/index.md b/openmetadata-docs/content/deployment/airflow/index.md index 12962189d02..2370f76179b 100644 --- a/openmetadata-docs/content/deployment/airflow/index.md +++ b/openmetadata-docs/content/deployment/airflow/index.md @@ -61,6 +61,16 @@ And then run the DAG as explained in each [Connector](/connectors). ### Airflow APIs + + +Note that these steps are required if you are reusing a host that already has Airflow installed. + +The `openmetadata-ingestion-apis` has a dependency on `apache-airflow>=2.2.2`. Please make sure that +your host satisfies such requirement. Only installing the `openmetadata-ingestion-apis` won't result +in a proper full Airflow installation. For that, please follow the Airflow [docs](https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html). + + + Goal: - Deploy metadata ingestion workflows directly from the UI. @@ -68,7 +78,7 @@ Goal: This process consists of three steps: 1. Install the APIs module, -2. Install the required plugins, and +2. Install the `openmetadata-ingestion` library and any extras you might need, and 3. Configure the OpenMetadata server. The goal of this module is to add some HTTP endpoints that the UI calls for deploying the Airflow DAGs. @@ -93,15 +103,38 @@ The OpenMetadata server takes all its configurations from a YAML file. You can f [...] airflowConfiguration: - apiEndpoint: http://${AIRFLOW_HOST:-localhost}:${AIRFLOW_PORT:-8080} - username: ${AIRFLOW_USERNAME:-admin} - password: ${AIRFLOW_PASSWORD:-admin} - metadataApiEndpoint: http://${SERVER_HOST:-localhost}:${SERVER_PORT:-8585}/api - authProvider: "no-auth" + apiEndpoint: ${AIRFLOW_HOST:-http://localhost:8080} + username: ${AIRFLOW_USERNAME:-admin} + password: ${AIRFLOW_PASSWORD:-admin} + metadataApiEndpoint: ${SERVER_HOST_API_URL:-http://localhost:8585/api} + authProvider: ${AIRFLOW_AUTH_PROVIDER:-"no-auth"} + +[...] ``` -Note that we also support picking up these values from environment variables, so you can safely set that up in the -machine hosting the OpenMetadata server. +If using Docker, make sure that you are passing the correct environment variables: + +```env +AIRFLOW_HOST: ${AIRFLOW_HOST:-http://ingestion:8080} +SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api} +``` + +#### Validating the installation + +What we need to verify here is that the OpenMetadata server can reach the Airflow APIs endpoints +(wherever they live: bare metal, containers, k8s pods...). One way to ensure that is to connect to the deployment +hosting your OpenMetadata server and running a query against the `/health` endpoint. For example: + +```bash +$ curl -XGET ${AIRFLOW_HOST}/api/v1/openmetadata/health +{"status": "healthy", "version": "x.y.z"} +``` + +It is important to do this validation passing the command as is (i.e., `curl -XGET ${AIRFLOW_HOST}/api/v1/openmetadata/health`) +and allowing the environment to do the substitution for you. That's the only way we can be sure that the setup is +correct. + +#### Adding SSO If you are running OpenMetadata with the security enabled, you can take a look at the server configuration for each security mode: