Fixes #15388: Use native backup tools (#15393)

* feat: use native backup tools

1. added mysqldump 8.3 to the ingestion container.
2. documented how to use native tools to back up and restore.
3. added deprecated message on the cli backup and restore.

* added deprecation notice for 1.3 backup

* removed 1.3.x deprecation notice

* added another backup page in 1.3 introducing SQL dump tools

* added --set-gtid-purged=OFF to the mysql dump process

docker/development/docker-compose-postgres.yml

@@ -49,6 +49,7 @@ services:
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
       - plugins.security.disabled=true
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=OpenMetadata_password123!!!
     networks:
       - local_app_net
     expose:
@@ -361,7 +362,7 @@ services:
       ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
       ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
       ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
-      ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
+      ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-"OpenMetadata_password123!!!"}
       SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"}
       ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
       ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}

docker/run_local_docker.sh

@@ -97,8 +97,11 @@ echo "Using ingestion dependency: ${INGESTION_DEPENDENCY:-all}"
 
 if [[ $database == "postgresql" ]]; then
     docker compose -f docker/development/docker-compose-postgres.yml build --build-arg INGESTION_DEPENDENCY="${INGESTION_DEPENDENCY:-all}" && docker compose -f docker/development/docker-compose-postgres.yml up -d
-else
+elif [[ $database == "mysql" ]]; then
     docker compose -f docker/development/docker-compose.yml build --build-arg INGESTION_DEPENDENCY="${INGESTION_DEPENDENCY:-all}" && docker compose -f docker/development/docker-compose.yml up -d
+else
+    echo "Invalid database type: $database"
+    exit 1
 fi
 
 RESULT=$?

ingestion/Dockerfile

@@ -1,3 +1,5 @@
+FROM mysql:8.3 as mysql
+
 FROM apache/airflow:2.7.3-python3.10
 USER root
 RUN curl -sS https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
@@ -38,6 +40,7 @@ RUN apt-get -qq update \
     # Accept MSSQL ODBC License
     && ACCEPT_EULA=Y apt-get install -y msodbcsql18 \
     && rm -rf /var/lib/apt/lists/*
+COPY --from=mysql /usr/bin/mysqldump /usr/bin/mysqldump
 
 RUN if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; \
  then \

ingestion/Dockerfile.ci

@@ -1,3 +1,5 @@
+FROM mysql:8.3 as mysql
+
 FROM apache/airflow:2.7.3-python3.10
 USER root
 RUN curl -sS https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
@@ -38,6 +40,7 @@ RUN apt-get -qq update \
     # Accept MSSQL ODBC License
     && ACCEPT_EULA=Y apt-get -qq install -y msodbcsql18 \
     && rm -rf /var/lib/apt/lists/*
+COPY --from=mysql /usr/bin/mysqldump /usr/bin/mysqldump
 
 RUN if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; \
  then \

ingestion/src/metadata/cli/backup.py

@@ -176,6 +176,13 @@ def run_backup(
     :param upload: URI to upload result file
 
     """
+    log_ansi_encoded_string(
+        color=ANSI.BRIGHT_RED,
+        bold=True,
+        message="WARNING: backup is deprecated starting 1.4.0. Use database native dump tools instead."
+        "For more information, please visit: "
+        "https://docs.open-metadata.org/v1.4.x/deployment/backup-restore-metadata",
+    )
     log_ansi_encoded_string(
         color=ANSI.GREEN,
         bold=False,

ingestion/src/metadata/cli/restore.py

@@ -71,6 +71,13 @@ def run_restore(
     :param common_restore_obj_instance: cls instance to fetch common args
     :param sql_file: local path of file to restore the backup
     """
+    log_ansi_encoded_string(
+        color=ANSI.BRIGHT_RED,
+        bold=True,
+        message="WARNING: restore is deprecated starting 1.4.0. Use database native tools to restore."
+        "For more information, please visit: "
+        "https://docs.open-metadata.org/v1.4.x/deployment/backup-restore-metadata",
+    )
     log_ansi_encoded_string(
         color=ANSI.GREEN,
         bold=False,

openmetadata-docs/content/v1.3.x/deployment/backup-restore-metadata-sql-dump-tools.md

@@ -0,0 +1,85 @@
+---
+title: Backup Metadata
+slug: /deployment/backup-restore-metadata-sql-dump-tools
+---
+
+# Backup & Restore Metadata
+
+## Introduction
+
+Since version 1.3.0, OpenMetadata's encourages using the builtin-tools for creating logical backups of the metadata:
+
+- `mysqldump` for MySQL
+- `pg_dump` for Postgres
+
+## Requirements
+
+- mysqldump 8.3 or higher (ingestion container is shipped with mysqldump 8.3)
+- pg_dump 13.3 or higher
+
+# Example
+
+Start a local instance of OpenMetadata using the `docker-compose` file provided in the repository. Then, we can use the following commands to backup the metadata:
+
+## MySQL
+
+### 1. Start a local docker deployment
+
+```shell
+docker/run_local_docker.sh
+```
+
+Ingest some data...
+
+### 2. Backup and Restore
+
+```shell
+BACKUP_FILE="backup_$(date +%Y%m%d%H%M).sql"
+DOCKER_COMPOSE_FILE="docker/development/docker-compose.yml"
+# backup
+docker compose -f $DOCKER_COMPOSE_FILE exec ingestion mysqldump --set-gtid-purged=OFF --no-tablespaces -u openmetadata_user -popenmetadata_password -h mysql -P 3306 openmetadata_db > $BACKUP_FILE
+# create the restore database
+docker compose -f $DOCKER_COMPOSE_FILE exec mysql  mysql -u root -ppassword -e "create database restore;"
+docker compose -f $DOCKER_COMPOSE_FILE exec mysql  mysql -u root -ppassword -e "grant all privileges on restore.* to 'openmetadata_user'@'%';"
+docker compose -f $DOCKER_COMPOSE_FILE exec mysql  mysql -u root -ppassword -e "flush privileges;"
+# restore from the backup
+docker compose -f $DOCKER_COMPOSE_FILE exec -T ingestion mysql -u openmetadata_user -popenmetadata_password -h mysql -P 3306 restore < $BACKUP_FILE
+```
+
+### 3. Restart the docker deployment with the restored database
+
+```shell
+export OM_DATABASE=restore
+docker compose -f $DOCKER_COMPOSE_FILE up -d
+```
+
+## PostgreSQL
+
+### 1. Start a local docker deployment
+
+```shell
+docker/run_local_docker.sh -d postgres
+```
+
+Ingest some data...
+
+### 2. Backup and Restore
+
+```shell
+BACKUP_FILE="backup_$(date +%Y%m%d%H%M).sql"
+DOCKER_COMPOSE_FILE="docker/development/docker-compose-postgres.yml"
+# backup
+docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password ingestion pg_dump -U openmetadata_user -h postgresql -d openmetadata_db > $BACKUP_FILE
+# create the restore database
+docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "create database restore;"
+docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "ALTER DATABASE restore OWNER TO openmetadata_user;"
+# restore from the backup
+docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password -T ingestion psql -U openmetadata_user -h postgresql -d restore < $BACKUP_FILE
+```
+
+### 3. Restart the docker deployment with the restored database
+
+```shell
+export OM_DATABASE=restore
+docker compose -f $DOCKER_COMPOSE_FILE up -d
+```

openmetadata-docs/content/v1.3.x/deployment/backup-restore-metadata.md

@@ -5,6 +5,18 @@ slug: /deployment/backup-restore-metadata
 
 # Backup & Restore Metadata
 
+## Using SQL Dump Tools
+
+{% inlineCalloutContainer %}
+  {% inlineCallout
+    color="violet-70"
+    icon="10k"
+    bold="Use Native SQL Backup Tools"
+    href="/deployment/backup-restore-metadata-sql-dump-tools" %}
+    Learn how to backup and restore metadata using native SQL backup tools.
+  {% /inlineCallout %}
+{% /inlineCalloutContainer %}
+
 ## Introduction
 
 The goal of OpenMetadata is to enable company-wide collaboration around metadata. The more we use it, the more value

openmetadata-docs/content/v1.4.x/deployment/backup-restore-metadata.md

@@ -0,0 +1,85 @@
+---
+title: Backup Metadata
+slug: /deployment/backup-restore-metadata
+---
+
+# Backup & Restore Metadata
+
+## Introduction
+
+Since version 1.4.0, OpenMetadata's encourages using the builtin-tools for creating logical backups of the metadata:
+
+- `mysqldump` for MySQL
+- `pg_dump` for Postgres
+
+## Requirements
+
+- mysqldump 8.3 or higher (ingestion container is shipped with mysqldump 8.3)
+- pg_dump 13.3 or higher
+
+# Example
+
+Start a local instance of OpenMetadata using the `docker-compose` file provided in the repository. Then, we can use the following commands to backup the metadata:
+
+## MySQL
+
+### 1. Start a local docker deployment
+
+```shell
+docker/run_local_docker.sh
+```
+
+Ingest some data...
+
+### 2. Backup and Restore
+
+```shell
+BACKUP_FILE="backup_$(date +%Y%m%d%H%M).sql"
+DOCKER_COMPOSE_FILE="docker/development/docker-compose.yml"
+# backup
+docker compose -f $DOCKER_COMPOSE_FILE exec ingestion mysqldump --no-tablespaces -u openmetadata_user -popenmetadata_password -h mysql -P 3306 openmetadata_db > $BACKUP_FILE
+# create the restore database
+docker compose -f $DOCKER_COMPOSE_FILE exec mysql  mysql -u root -ppassword -e "create database restore;"
+docker compose -f $DOCKER_COMPOSE_FILE exec mysql  mysql -u root -ppassword -e "grant all privileges on restore.* to 'openmetadata_user'@'%';"
+docker compose -f $DOCKER_COMPOSE_FILE exec mysql  mysql -u root -ppassword -e "flush privileges;"
+# restore from the backup
+docker compose -f $DOCKER_COMPOSE_FILE exec -T ingestion mysql -u openmetadata_user -popenmetadata_password -h mysql -P 3306 restore < $BACKUP_FILE
+```
+
+### 3. Restart the docker deployment with the restored database
+
+```shell
+export OM_DATABASE=restore
+docker compose -f $DOCKER_COMPOSE_FILE up -d
+```
+
+## PostgreSQL
+
+### 1. Start a local docker deployment
+
+```shell
+docker/run_local_docker.sh -d postgres
+```
+
+Ingest some data...
+
+### 2. Backup and Restore
+
+```shell
+BACKUP_FILE="backup_$(date +%Y%m%d%H%M).sql"
+DOCKER_COMPOSE_FILE="docker/development/docker-compose-postgres.yml"
+# backup
+docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password ingestion pg_dump -U openmetadata_user -h postgresql -d openmetadata_db > $BACKUP_FILE
+# create the restore database
+docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "create database restore;"
+docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "ALTER DATABASE restore OWNER TO openmetadata_user;"
+# restore from the backup
+docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password -T ingestion psql -U openmetadata_user -h postgresql -d restore < $BACKUP_FILE
+```
+
+### 3. Restart the docker deployment with the restored database
+
+```shell
+export OM_DATABASE=restore
+docker compose -f $DOCKER_COMPOSE_FILE up -d
+```