From 0d74cf7029133a5a00f358eb082d0bf4893e1ed5 Mon Sep 17 00:00:00 2001 From: Akash Jain <15995028+akash-jain-10@users.noreply.github.com> Date: Thu, 19 Jan 2023 14:54:08 +0530 Subject: [PATCH] Fix/docs-misc (#9666) * use latest version 0.13.1 in examples * fix docs for #9654 * fix for #9607 --- openmetadata-docs/content/connectors/custom-connectors.md | 2 +- .../connectors/ingestion/deployment/airflow-dag.md | 4 ++-- .../connectors/ingestion/deployment/running-from-mwaa.md | 2 +- .../content/connectors/pipeline/airflow/gcs.md | 6 +++--- openmetadata-docs/content/deployment/docker/index.md | 8 ++++---- openmetadata-docs/content/deployment/kubernetes/eks.md | 8 +++++--- openmetadata-docs/content/deployment/kubernetes/values.md | 4 +++- openmetadata-docs/content/deployment/upgrade/docker.md | 2 +- openmetadata-docs/content/deployment/upgrade/index.md | 8 ++++---- .../build-code-and-run-tests/openmetadata-server.md | 4 ++-- 10 files changed, 26 insertions(+), 22 deletions(-) diff --git a/openmetadata-docs/content/connectors/custom-connectors.md b/openmetadata-docs/content/connectors/custom-connectors.md index fbcd5bc2b41..828c2f06e25 100644 --- a/openmetadata-docs/content/connectors/custom-connectors.md +++ b/openmetadata-docs/content/connectors/custom-connectors.md @@ -54,7 +54,7 @@ We will be running the demo against the OpenMetadata version `0.12.2`, therefore ```Dockerfile # Base image from the right version -FROM openmetadata/ingestion:0.12.2 +FROM openmetadata/ingestion:0.13.1 # Let's use the same workdir as the ingestion image WORKDIR ingestion diff --git a/openmetadata-docs/content/connectors/ingestion/deployment/airflow-dag.md b/openmetadata-docs/content/connectors/ingestion/deployment/airflow-dag.md index 4dc2345d5ec..61d0a7e9592 100644 --- a/openmetadata-docs/content/connectors/ingestion/deployment/airflow-dag.md +++ b/openmetadata-docs/content/connectors/ingestion/deployment/airflow-dag.md @@ -121,7 +121,7 @@ with models.DAG( ) as dag: DockerOperator( command="python main.py", - image="openmetadata/ingestion-base:0.12.1", + image="openmetadata/ingestion-base:0.13.1", environment={"config": config, "pipelineType": "metadata"}, docker_url="unix://var/run/docker.sock", # To allow to start Docker. Needs chmod 666 permissions tty=True, @@ -138,7 +138,7 @@ Make sure to tune out the DAG configurations (`schedule_interval`, `start_date`, -Note that the example uses the image `openmetadata/ingestion-base:0.12.1`. Update that accordingly for higher version +Note that the example uses the image `openmetadata/ingestion-base:0.13.1`. Update that accordingly for higher version once they are released. Also, the image version should be aligned with your OpenMetadata server version to avoid incompatibilities. diff --git a/openmetadata-docs/content/connectors/ingestion/deployment/running-from-mwaa.md b/openmetadata-docs/content/connectors/ingestion/deployment/running-from-mwaa.md index 7fbdd2fc993..220151228b2 100644 --- a/openmetadata-docs/content/connectors/ingestion/deployment/running-from-mwaa.md +++ b/openmetadata-docs/content/connectors/ingestion/deployment/running-from-mwaa.md @@ -132,7 +132,7 @@ We will now describe the steps, following the official AWS documentation. - The cluster just needs a task to run in `FARGATE` mode. - The required image is `registry.hub.docker.com/openmetadata/ingestion-base:x.y.z` - - The same logic as above applies. The `x.y.z` version needs to match the server version. For example, `registry.hub.docker.com/openmetadata/ingestion-base:0.12.2` + - The same logic as above applies. The `x.y.z` version needs to match the server version. For example, `registry.hub.docker.com/openmetadata/ingestion-base:0.13.1` We have tested this process with a Task Memory of 512MB and Task CPU (unit) of 256. This can be tuned depending on the amount of metadata that needs to be ingested. diff --git a/openmetadata-docs/content/connectors/pipeline/airflow/gcs.md b/openmetadata-docs/content/connectors/pipeline/airflow/gcs.md index 8b8eb5a265e..8b95d4f8375 100644 --- a/openmetadata-docs/content/connectors/pipeline/airflow/gcs.md +++ b/openmetadata-docs/content/connectors/pipeline/airflow/gcs.md @@ -223,7 +223,7 @@ with models.DAG( task_id="ingest", name="ingest", cmds=["python", "main.py"], - image="openmetadata/ingestion-base:0.12.1", + image="openmetadata/ingestion-base:0.13.1", namespace='default', env_vars={"config": config, "pipelineType": "metadata"}, dag=dag, @@ -238,7 +238,7 @@ You can name the task as you want (`task_id` and `name`). The important points h be changed, and the `env_vars`. The `main.py` script that gets shipped within the image will load the env vars as they are shown, so only modify the content of the config YAML, but not this dictionary. -Note that the example uses the image `openmetadata/ingestion-base:0.12.1`. Update that accordingly for higher version +Note that the example uses the image `openmetadata/ingestion-base:0.13.1`. Update that accordingly for higher version once they are released. Also, the image version should be aligned with your OpenMetadata server version to avoid incompatibilities. @@ -247,7 +247,7 @@ KubernetesPodOperator( task_id="ingest", name="ingest", cmds=["python", "main.py"], - image="openmetadata/ingestion-base:0.12.1", + image="openmetadata/ingestion-base:0.13.1", namespace='default', env_vars={"config": config, "pipelineType": "metadata"}, dag=dag, diff --git a/openmetadata-docs/content/deployment/docker/index.md b/openmetadata-docs/content/deployment/docker/index.md index 49ed0cde826..ff3826e9bec 100644 --- a/openmetadata-docs/content/deployment/docker/index.md +++ b/openmetadata-docs/content/deployment/docker/index.md @@ -97,10 +97,10 @@ You can validate that all containers are up by running with command `docker ps`. ```commandline ❯ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -470cc8149826 openmetadata/server:0.12.2 "./openmetadata-star…" 45 seconds ago Up 43 seconds 3306/tcp, 9200/tcp, 9300/tcp, 0.0.0.0:8585-8586->8585-8586/tcp openmetadata_server -63578aacbff5 openmetadata/ingestion:0.12.2 "./ingestion_depende…" 45 seconds ago Up 43 seconds 0.0.0.0:8080->8080/tcp openmetadata_ingestion +470cc8149826 openmetadata/server:0.13.1 "./openmetadata-star…" 45 seconds ago Up 43 seconds 3306/tcp, 9200/tcp, 9300/tcp, 0.0.0.0:8585-8586->8585-8586/tcp openmetadata_server +63578aacbff5 openmetadata/ingestion:0.13.1 "./ingestion_depende…" 45 seconds ago Up 43 seconds 0.0.0.0:8080->8080/tcp openmetadata_ingestion 9f5ee8334f4b docker.elastic.co/elasticsearch/elasticsearch:7.10.2 "/tini -- /usr/local…" 45 seconds ago Up 44 seconds 0.0.0.0:9200->9200/tcp, 0.0.0.0:9300->9300/tcp openmetadata_elasticsearch -08947ab3424b openmetadata/db:0.12.2 "/entrypoint.sh mysq…" 45 seconds ago Up 44 seconds (healthy) 3306/tcp, 33060-33061/tcp openmetadata_mysql +08947ab3424b openmetadata/db:0.13.1 "/entrypoint.sh mysq…" 45 seconds ago Up 44 seconds (healthy) 3306/tcp, 33060-33061/tcp openmetadata_mysql ``` In a few seconds, you should be able to access the OpenMetadata UI at [http://localhost:8585](http://localhost:8585) @@ -148,7 +148,7 @@ If you are planning on going to PROD, we recommend to validate below points: - MySQL and OpenSearch (ElasticSearch) are available. - OpenMetadata-Server require the minimum configuration of 2vCPU and 6Memory (GiB) - OpenMetadata-Ingestion require the minimum configuration of 2vCPU and 8Memory (GiB) -- We also recommend to bind Docker Volumes for data persistence. Learn how to do so [here](/deployment/docker/volumes). +- We also recommend to bind Docker Volumes for data persistence. Minimum disck space required would be 128 Gib. Learn how to do so [here](/deployment/docker/volumes). ### Steps for Deploying Ingestion - Download the docker-compose.yml file from the release page [here](https://github.com/open-metadata/OpenMetadata/releases). - Update the environment variables below for OpenMetadata-Ingestion Docker Compose backed systems to connect with Database. diff --git a/openmetadata-docs/content/deployment/kubernetes/eks.md b/openmetadata-docs/content/deployment/kubernetes/eks.md index 123c1b1e974..c1a34fd1151 100644 --- a/openmetadata-docs/content/deployment/kubernetes/eks.md +++ b/openmetadata-docs/content/deployment/kubernetes/eks.md @@ -35,6 +35,8 @@ The below guide provides Persistent Volumes provisioning as static volumes (mean ## Provision EFS backed PVs, PVCs for Airflow DAGs and Airflow Logs +Please note that we are using one AWS Elastic File System (EFS) service with sub-directories as `airflow-dags` and `airflow-logs` with the reference in this documentation. Also, it is presumed that `airflow-dags` and `airflow-logs` directories are already available on that file system. + ```yaml @@ -47,7 +49,7 @@ metadata: app: airflow-dags spec: capacity: - storage: 5Gi + storage: 10Gi storageClassName: "" accessModes: - ReadWriteMany @@ -70,7 +72,7 @@ spec: storageClassName: "" resources: requests: - storage: 5Gi + storage: 10Gi ``` Create Persistent Volumes and Persistent Volume claims with the below command. @@ -116,7 +118,7 @@ spec: storageClassName: "" resources: requests: - storage: 10Gi + storage: 5Gi ``` Create Persistent Volumes and Persistent Volume claims with the below command. diff --git a/openmetadata-docs/content/deployment/kubernetes/values.md b/openmetadata-docs/content/deployment/kubernetes/values.md index 92c0b0dc2e0..67a2bd22dd4 100644 --- a/openmetadata-docs/content/deployment/kubernetes/values.md +++ b/openmetadata-docs/content/deployment/kubernetes/values.md @@ -59,7 +59,9 @@ This page list all the supported helm values for OpenMetadata Helm Charts. | global.elasticsearch.trustStore.path | string | `Empty String` | | global.elasticsearch.trustStore.password.secretRef | string | `elasticsearch-truststore-secrets` | | global.elasticsearch.trustStore.password.secretKey | string | `openmetadata-elasticsearch-truststore-password` | -| global.fernetKey | string | `jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=` | +| global.fernetkey.value | string | `jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=` | +| global.fernetkey.secretRef | string | `` | +| global.fernetkey.secretKef | string | `` | | global.jwtTokenConfiguration.enabled | bool | `true` | | global.jwtTokenConfiguration.rsapublicKeyFilePath | string | `./conf/public_key.der` | | global.jwtTokenConfiguration.rsaprivateKeyFilePath | string | `./conf/private_key.der` | diff --git a/openmetadata-docs/content/deployment/upgrade/docker.md b/openmetadata-docs/content/deployment/upgrade/docker.md index cd83838eee5..0d8db7cc844 100644 --- a/openmetadata-docs/content/deployment/upgrade/docker.md +++ b/openmetadata-docs/content/deployment/upgrade/docker.md @@ -82,7 +82,7 @@ This step is only required if you have not done that yet. ``` mysql: container_name: openmetadata_mysql - image: openmetadata/db:0.12.3 + image: openmetadata/db:0.13.1 restart: always environment: MYSQL_ROOT_PASSWORD: password diff --git a/openmetadata-docs/content/deployment/upgrade/index.md b/openmetadata-docs/content/deployment/upgrade/index.md index 86d684f1b0b..bb886740b38 100644 --- a/openmetadata-docs/content/deployment/upgrade/index.md +++ b/openmetadata-docs/content/deployment/upgrade/index.md @@ -118,7 +118,7 @@ with openmetadata/ingestion docker image which will be used with docker compose - There is no change or effect with docker installation - This is a breaking change if you are using a custom openmetadata-dependencies kubernetes helm chart values file. -You will need to manually update the airflow image and tag with openmetadata/ingestion:0.12.1 +You will need to manually update the airflow image and tag with openmetadata/ingestion:0.13.1 ```yaml ... @@ -126,17 +126,17 @@ airflow: airflow: image: repository: openmetadata/ingestion - tag: 0.12.1 + tag: 0.13.1 pullPolicy: "IfNotPresent" ... ```

-If you are extending openmetadata/airflow docker image with 0.12.0 release, you can safely replace that with openmetadata/ingestion:0.12.1 Docker Image. +If you are extending openmetadata/airflow docker image with 0.13.1 release, you can safely replace that with openmetadata/ingestion:0.13.1 Docker Image.

```Dockerfile -FROM openmetadata/ingestion:0.12.1 +FROM openmetadata/ingestion:0.13.1 USER airflow ... ``` diff --git a/openmetadata-docs/content/developers/contribute/build-code-and-run-tests/openmetadata-server.md b/openmetadata-docs/content/developers/contribute/build-code-and-run-tests/openmetadata-server.md index d6fc0b47325..06148e27003 100644 --- a/openmetadata-docs/content/developers/contribute/build-code-and-run-tests/openmetadata-server.md +++ b/openmetadata-docs/content/developers/contribute/build-code-and-run-tests/openmetadata-server.md @@ -11,13 +11,13 @@ Learn how to run the OpenMetadata server in development mode by using Docker and - For an easy install of MySQL and ES, just install Docker on your local machine and run the following commands from the top-level directory ```shell -docker compose -f docker/local-metadata/docker-compose.yml -d up mysql elasticsearch +docker compose -f docker/local-metadata/docker-compose.yml up mysql elasticsearch --build -d ``` - For an easy install of PostgreSQL and ES, just install Docker on your local machine and run the following commands from the top-level directory ```shell -docker compose -f docker/local-metadata/docker-compose-postgres.yml -d up postgresql elasticsearch +docker compose -f docker/local-metadata/docker-compose-postgres.yml up postgresql elasticsearch --build -d ``` - Bootstrap MySQL with tables