diff --git a/docker/development/docker-compose-postgres.yml b/docker/development/docker-compose-postgres.yml index b742b6104a7..01eaf3cb07f 100644 --- a/docker/development/docker-compose-postgres.yml +++ b/docker/development/docker-compose-postgres.yml @@ -49,6 +49,7 @@ services: - discovery.type=single-node - ES_JAVA_OPTS=-Xms1024m -Xmx1024m - plugins.security.disabled=true + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=OpenMetadata_password123!!! networks: - local_app_net expose: @@ -361,7 +362,7 @@ services: ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200} ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http} ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""} - ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""} + ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-"OpenMetadata_password123!!!"} SEARCH_TYPE: ${SEARCH_TYPE:- "opensearch"} ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""} ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""} diff --git a/docker/run_local_docker.sh b/docker/run_local_docker.sh index 9174d9f9f40..06e19c6c63a 100755 --- a/docker/run_local_docker.sh +++ b/docker/run_local_docker.sh @@ -97,8 +97,11 @@ echo "Using ingestion dependency: ${INGESTION_DEPENDENCY:-all}" if [[ $database == "postgresql" ]]; then docker compose -f docker/development/docker-compose-postgres.yml build --build-arg INGESTION_DEPENDENCY="${INGESTION_DEPENDENCY:-all}" && docker compose -f docker/development/docker-compose-postgres.yml up -d -else +elif [[ $database == "mysql" ]]; then docker compose -f docker/development/docker-compose.yml build --build-arg INGESTION_DEPENDENCY="${INGESTION_DEPENDENCY:-all}" && docker compose -f docker/development/docker-compose.yml up -d +else + echo "Invalid database type: $database" + exit 1 fi RESULT=$? diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index a895e68bafc..87e3eaf6141 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -1,3 +1,5 @@ +FROM mysql:8.3 as mysql + FROM apache/airflow:2.7.3-python3.10 USER root RUN curl -sS https://packages.microsoft.com/keys/microsoft.asc | apt-key add - @@ -38,6 +40,7 @@ RUN apt-get -qq update \ # Accept MSSQL ODBC License && ACCEPT_EULA=Y apt-get install -y msodbcsql18 \ && rm -rf /var/lib/apt/lists/* +COPY --from=mysql /usr/bin/mysqldump /usr/bin/mysqldump RUN if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; \ then \ diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci index 6edca5365a2..8268065aa07 100644 --- a/ingestion/Dockerfile.ci +++ b/ingestion/Dockerfile.ci @@ -1,3 +1,5 @@ +FROM mysql:8.3 as mysql + FROM apache/airflow:2.7.3-python3.10 USER root RUN curl -sS https://packages.microsoft.com/keys/microsoft.asc | apt-key add - @@ -38,6 +40,7 @@ RUN apt-get -qq update \ # Accept MSSQL ODBC License && ACCEPT_EULA=Y apt-get -qq install -y msodbcsql18 \ && rm -rf /var/lib/apt/lists/* +COPY --from=mysql /usr/bin/mysqldump /usr/bin/mysqldump RUN if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; \ then \ diff --git a/ingestion/src/metadata/cli/backup.py b/ingestion/src/metadata/cli/backup.py index 179c8d0535e..94441a723eb 100644 --- a/ingestion/src/metadata/cli/backup.py +++ b/ingestion/src/metadata/cli/backup.py @@ -176,6 +176,13 @@ def run_backup( :param upload: URI to upload result file """ + log_ansi_encoded_string( + color=ANSI.BRIGHT_RED, + bold=True, + message="WARNING: backup is deprecated starting 1.4.0. Use database native dump tools instead." + "For more information, please visit: " + "https://docs.open-metadata.org/v1.4.x/deployment/backup-restore-metadata", + ) log_ansi_encoded_string( color=ANSI.GREEN, bold=False, diff --git a/ingestion/src/metadata/cli/restore.py b/ingestion/src/metadata/cli/restore.py index da10426c4ef..c06bd92c48b 100644 --- a/ingestion/src/metadata/cli/restore.py +++ b/ingestion/src/metadata/cli/restore.py @@ -71,6 +71,13 @@ def run_restore( :param common_restore_obj_instance: cls instance to fetch common args :param sql_file: local path of file to restore the backup """ + log_ansi_encoded_string( + color=ANSI.BRIGHT_RED, + bold=True, + message="WARNING: restore is deprecated starting 1.4.0. Use database native tools to restore." + "For more information, please visit: " + "https://docs.open-metadata.org/v1.4.x/deployment/backup-restore-metadata", + ) log_ansi_encoded_string( color=ANSI.GREEN, bold=False, diff --git a/openmetadata-docs/content/v1.3.x/deployment/backup-restore-metadata-sql-dump-tools.md b/openmetadata-docs/content/v1.3.x/deployment/backup-restore-metadata-sql-dump-tools.md new file mode 100644 index 00000000000..23550f6aab8 --- /dev/null +++ b/openmetadata-docs/content/v1.3.x/deployment/backup-restore-metadata-sql-dump-tools.md @@ -0,0 +1,85 @@ +--- +title: Backup Metadata +slug: /deployment/backup-restore-metadata-sql-dump-tools +--- + +# Backup & Restore Metadata + +## Introduction + +Since version 1.3.0, OpenMetadata's encourages using the builtin-tools for creating logical backups of the metadata: + +- `mysqldump` for MySQL +- `pg_dump` for Postgres + +## Requirements + +- mysqldump 8.3 or higher (ingestion container is shipped with mysqldump 8.3) +- pg_dump 13.3 or higher + +# Example + +Start a local instance of OpenMetadata using the `docker-compose` file provided in the repository. Then, we can use the following commands to backup the metadata: + +## MySQL + +### 1. Start a local docker deployment + +```shell +docker/run_local_docker.sh +``` + +Ingest some data... + +### 2. Backup and Restore + +```shell +BACKUP_FILE="backup_$(date +%Y%m%d%H%M).sql" +DOCKER_COMPOSE_FILE="docker/development/docker-compose.yml" +# backup +docker compose -f $DOCKER_COMPOSE_FILE exec ingestion mysqldump --set-gtid-purged=OFF --no-tablespaces -u openmetadata_user -popenmetadata_password -h mysql -P 3306 openmetadata_db > $BACKUP_FILE +# create the restore database +docker compose -f $DOCKER_COMPOSE_FILE exec mysql mysql -u root -ppassword -e "create database restore;" +docker compose -f $DOCKER_COMPOSE_FILE exec mysql mysql -u root -ppassword -e "grant all privileges on restore.* to 'openmetadata_user'@'%';" +docker compose -f $DOCKER_COMPOSE_FILE exec mysql mysql -u root -ppassword -e "flush privileges;" +# restore from the backup +docker compose -f $DOCKER_COMPOSE_FILE exec -T ingestion mysql -u openmetadata_user -popenmetadata_password -h mysql -P 3306 restore < $BACKUP_FILE +``` + +### 3. Restart the docker deployment with the restored database + +```shell +export OM_DATABASE=restore +docker compose -f $DOCKER_COMPOSE_FILE up -d +``` + +## PostgreSQL + +### 1. Start a local docker deployment + +```shell +docker/run_local_docker.sh -d postgres +``` + +Ingest some data... + +### 2. Backup and Restore + +```shell +BACKUP_FILE="backup_$(date +%Y%m%d%H%M).sql" +DOCKER_COMPOSE_FILE="docker/development/docker-compose-postgres.yml" +# backup +docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password ingestion pg_dump -U openmetadata_user -h postgresql -d openmetadata_db > $BACKUP_FILE +# create the restore database +docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "create database restore;" +docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "ALTER DATABASE restore OWNER TO openmetadata_user;" +# restore from the backup +docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password -T ingestion psql -U openmetadata_user -h postgresql -d restore < $BACKUP_FILE +``` + +### 3. Restart the docker deployment with the restored database + +```shell +export OM_DATABASE=restore +docker compose -f $DOCKER_COMPOSE_FILE up -d +``` diff --git a/openmetadata-docs/content/v1.3.x/deployment/backup-restore-metadata.md b/openmetadata-docs/content/v1.3.x/deployment/backup-restore-metadata.md index e2802455b56..c8c061706d0 100644 --- a/openmetadata-docs/content/v1.3.x/deployment/backup-restore-metadata.md +++ b/openmetadata-docs/content/v1.3.x/deployment/backup-restore-metadata.md @@ -5,6 +5,18 @@ slug: /deployment/backup-restore-metadata # Backup & Restore Metadata +## Using SQL Dump Tools + +{% inlineCalloutContainer %} + {% inlineCallout + color="violet-70" + icon="10k" + bold="Use Native SQL Backup Tools" + href="/deployment/backup-restore-metadata-sql-dump-tools" %} + Learn how to backup and restore metadata using native SQL backup tools. + {% /inlineCallout %} +{% /inlineCalloutContainer %} + ## Introduction The goal of OpenMetadata is to enable company-wide collaboration around metadata. The more we use it, the more value diff --git a/openmetadata-docs/content/v1.4.x/deployment/backup-restore-metadata.md b/openmetadata-docs/content/v1.4.x/deployment/backup-restore-metadata.md new file mode 100644 index 00000000000..1355e4947a8 --- /dev/null +++ b/openmetadata-docs/content/v1.4.x/deployment/backup-restore-metadata.md @@ -0,0 +1,85 @@ +--- +title: Backup Metadata +slug: /deployment/backup-restore-metadata +--- + +# Backup & Restore Metadata + +## Introduction + +Since version 1.4.0, OpenMetadata's encourages using the builtin-tools for creating logical backups of the metadata: + +- `mysqldump` for MySQL +- `pg_dump` for Postgres + +## Requirements + +- mysqldump 8.3 or higher (ingestion container is shipped with mysqldump 8.3) +- pg_dump 13.3 or higher + +# Example + +Start a local instance of OpenMetadata using the `docker-compose` file provided in the repository. Then, we can use the following commands to backup the metadata: + +## MySQL + +### 1. Start a local docker deployment + +```shell +docker/run_local_docker.sh +``` + +Ingest some data... + +### 2. Backup and Restore + +```shell +BACKUP_FILE="backup_$(date +%Y%m%d%H%M).sql" +DOCKER_COMPOSE_FILE="docker/development/docker-compose.yml" +# backup +docker compose -f $DOCKER_COMPOSE_FILE exec ingestion mysqldump --no-tablespaces -u openmetadata_user -popenmetadata_password -h mysql -P 3306 openmetadata_db > $BACKUP_FILE +# create the restore database +docker compose -f $DOCKER_COMPOSE_FILE exec mysql mysql -u root -ppassword -e "create database restore;" +docker compose -f $DOCKER_COMPOSE_FILE exec mysql mysql -u root -ppassword -e "grant all privileges on restore.* to 'openmetadata_user'@'%';" +docker compose -f $DOCKER_COMPOSE_FILE exec mysql mysql -u root -ppassword -e "flush privileges;" +# restore from the backup +docker compose -f $DOCKER_COMPOSE_FILE exec -T ingestion mysql -u openmetadata_user -popenmetadata_password -h mysql -P 3306 restore < $BACKUP_FILE +``` + +### 3. Restart the docker deployment with the restored database + +```shell +export OM_DATABASE=restore +docker compose -f $DOCKER_COMPOSE_FILE up -d +``` + +## PostgreSQL + +### 1. Start a local docker deployment + +```shell +docker/run_local_docker.sh -d postgres +``` + +Ingest some data... + +### 2. Backup and Restore + +```shell +BACKUP_FILE="backup_$(date +%Y%m%d%H%M).sql" +DOCKER_COMPOSE_FILE="docker/development/docker-compose-postgres.yml" +# backup +docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password ingestion pg_dump -U openmetadata_user -h postgresql -d openmetadata_db > $BACKUP_FILE +# create the restore database +docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "create database restore;" +docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "ALTER DATABASE restore OWNER TO openmetadata_user;" +# restore from the backup +docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password -T ingestion psql -U openmetadata_user -h postgresql -d restore < $BACKUP_FILE +``` + +### 3. Restart the docker deployment with the restored database + +```shell +export OM_DATABASE=restore +docker compose -f $DOCKER_COMPOSE_FILE up -d +```