From f3855a19096e361bff1ef200a616de087f41a07d Mon Sep 17 00:00:00 2001 From: Vj-L <102945356+Vj-L@users.noreply.github.com> Date: Wed, 21 Dec 2022 23:59:49 +0530 Subject: [PATCH] Added the default volumes for ElasticSearch (#9452) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added the default volumes for ElasticSearch * Added docs for ES Volumes * Updated as per the comments * Updated as per the comments * Added the docs for bare-metal upgrade * Apply suggestions from code review * Update docker/local-metadata/docker-compose.yml * Pylint check addressed Co-authored-by: “Vijay” <“vijay.l@deuexsolutions.com”> Co-authored-by: Nahuel --- docker/metadata/docker-compose-postgres.yml | 2 ++ docker/metadata/docker-compose.yml | 2 ++ docker/run_local_docker.sh | 1 - ingestion/src/metadata/cli/docker.py | 3 +-- .../content/deployment/docker/volumes.md | 13 +++++++++++++ .../content/deployment/upgrade/bare-metal.md | 19 +++++++++++++++++++ .../content/deployment/upgrade/index.md | 9 ++++++++- 7 files changed, 45 insertions(+), 4 deletions(-) diff --git a/docker/metadata/docker-compose-postgres.yml b/docker/metadata/docker-compose-postgres.yml index a89663c0525..52cc1188fce 100644 --- a/docker/metadata/docker-compose-postgres.yml +++ b/docker/metadata/docker-compose-postgres.yml @@ -48,6 +48,8 @@ services: ports: - "9200:9200" - "9300:9300" + volumes: + - ./docker-volume/es-data:/usr/share/elasticsearch/data openmetadata-server: container_name: openmetadata_server diff --git a/docker/metadata/docker-compose.yml b/docker/metadata/docker-compose.yml index b34e410c20c..e9a12f18ad3 100644 --- a/docker/metadata/docker-compose.yml +++ b/docker/metadata/docker-compose.yml @@ -46,6 +46,8 @@ services: ports: - "9200:9200" - "9300:9300" + volumes: + - ./docker-volume/es-data:/usr/share/elasticsearch/data openmetadata-server: container_name: openmetadata_server diff --git a/docker/run_local_docker.sh b/docker/run_local_docker.sh index 9f7f27ed879..6aba46e17c5 100755 --- a/docker/run_local_docker.sh +++ b/docker/run_local_docker.sh @@ -87,7 +87,6 @@ docker compose -f docker/local-metadata/docker-compose-postgres.yml down docker compose -f docker/local-metadata/docker-compose.yml down echo "Starting Local Docker Containers" -mkdir -p docker-volume && mkdir -p docker-volume/db-data echo "Using ingestion dependency: ${INGESTION_DEPENDENCY:-all}" if [[ $database == "postgresql" ]]; then diff --git a/ingestion/src/metadata/cli/docker.py b/ingestion/src/metadata/cli/docker.py index d628ef5836e..cc2237d7121 100644 --- a/ingestion/src/metadata/cli/docker.py +++ b/ingestion/src/metadata/cli/docker.py @@ -72,8 +72,7 @@ def docker_volume(): # create a main directory if not os.path.exists(MAIN_DIR): os.mkdir(MAIN_DIR) - db = "db-data" - path_to_join = [db] + path_to_join = ["db-data", "es-data"] final_path = [] for path in path_to_join: temp_path = os.path.join(MAIN_DIR, path) diff --git a/openmetadata-docs/content/deployment/docker/volumes.md b/openmetadata-docs/content/deployment/docker/volumes.md index ca209ede158..2af0fab5f1f 100644 --- a/openmetadata-docs/content/deployment/docker/volumes.md +++ b/openmetadata-docs/content/deployment/docker/volumes.md @@ -41,7 +41,20 @@ services: - ./docker-volume/db:/var/lib/postgresql/data ... ``` +## Volumes for Elasticsearch container: +Following are the changes we have to do while mounting the directory for Elasticsearch in OpenMetadata. +- Update or add the volume in the docker-compose.yml file. +Open the file `docker-compose.yml` downloaded from the Release page [Link](https://github.com/open-metadata/OpenMetadata/releases/download/0.13.0-release/docker-compose.yml) . +```commandline +version: "3.9" +services: + elasticsearch: + ... + volumes: + - ./docker-volume/es-data:/usr/share/elasticsearch/data + ... +``` ## Volumes for ingestion container Following are the changes we have to do while mounting the directory for ingestion in OpenMetadata. Here we will maintaing different directory for dag_generated_configs, dags and secrets. - Remove the below section from the docker-compose.yml file. diff --git a/openmetadata-docs/content/deployment/upgrade/bare-metal.md b/openmetadata-docs/content/deployment/upgrade/bare-metal.md index 01b871bcd8d..20dbb942d51 100644 --- a/openmetadata-docs/content/deployment/upgrade/bare-metal.md +++ b/openmetadata-docs/content/deployment/upgrade/bare-metal.md @@ -19,7 +19,26 @@ This guide assumes that you have an OpenMetadata deployment that you installed a It is adviced to go through [openmetadata release notes](/deployment/upgrade/versions/012-to-013) before starting the upgrade process. +### Backup 0.12.3 data +1. Make sure your instance is connected to the Database server +2. Create a virtual environment to install an upgraded `metadata` version to run the backup command: + 1. `python -m venv venv` + 2. `source venv/bin/activate` + 3. `PIP_USER=false pip install openmetadata-ingestion~=0.13.1` +3. Validate the installed `metadata` version with `python -m metadata --version`, which should tell us that we are + indeed at 0.13.1. Notice the `python -m metadata` vs. `metadata`. +4. Run the backup using the updated `metadata` CLI: + ``` + python -m metadata backup -u openmetadata_user -p openmetadata_password -H mysql -d openmetadata_db --port 3306 + ``` + if using Postgres: + ``` + python -m metadata backup -u openmetadata_user -p openmetadata_password -H postgresql -d openmetadata_db --port 5432 -s public + ``` +5. This will generate the .sql file which can be used for the backup + In our case, the backup file was named `openmetadata_202212201528_backup.sql`. You can copy the name from the backup + command output. ### 1. Download the binaries for the release you want to install OpenMetadata release binaries are maintained as GitHub releases. diff --git a/openmetadata-docs/content/deployment/upgrade/index.md b/openmetadata-docs/content/deployment/upgrade/index.md index a114fc74f9f..080bd9fe1af 100644 --- a/openmetadata-docs/content/deployment/upgrade/index.md +++ b/openmetadata-docs/content/deployment/upgrade/index.md @@ -29,7 +29,14 @@ If you are upgrading production this is the recommended version to upgrade. ## Breaking Changes from 0.13.0 Feature Release -OpenMetadata Release 0.13.0 introduces below breaking changes - +OpenMetadata Release 0.13.x introduces below breaking changes: + +### Docker Volumes + OpenMetadata Release 0.13.x introduces Default Docker Volumes for Databse(MYQSL, PostgreSQL) and ElasticSearch with Docker deployment. + + - If you are looking for the fresh deployment of 0.13.x - [here](https://docs.open-metadata.org/deployment/docker) + - If you are looking for upgrading of the new version i.e 0.13.x - [here](https://docs.open-metadata.org/deployment/upgrade/docker) + ### dbt Workflow dbt ingestion has been separated from the metadata ingestion. It can now be configured as a separate workflow after completing the metadata ingestion workflow.