yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-27 15:38:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

[FEAT] Implement Docker InitContainer for migration (#12714)

Browse Source 
* [FEAT] Implement Docker InitContainer for migration

* [FEAT] Implement Docker InitContainer for migration

* [FEAT] Implement Docker InitContainer for migration

* [FEAT] Implement Docker InitContainer for migration

---------

Co-authored-by: Dhruvin Maniar <dhruvinmaniar@192.168.1.27>
Co-authored-by: Dhruvin Maniar <dhruvinmaniar@Dhruvins-MacBook-Pro.local>
Co-authored-by: Akash Jain <15995028+akash-jain-10@users.noreply.github.com>
This commit is contained in:
Dhruvin Maniar 2023-08-03 16:01:20 +05:30 committed by GitHub
parent 10f2567fe9
commit 403e407478
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 787 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
docker/development/docker-compose-postgres.yml
View File

@ -59,6 +59,160 @@ services:
volumes:
- es-data:/usr/share/elasticsearch/data
execute-migrate-all:
build:
context: ../../.
dockerfile: docker/development/Dockerfile
container_name: execute_migrate_all
command: "./bootstrap/bootstrap_storage.sh migrate-all"
environment:
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
SERVER_PORT: ${SERVER_PORT:-8585}
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
OPENMETADATA_DEBUG: ${OPENMETADATA_DEBUG:-false}
# Migration
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
# OpenMetadata Server Authentication Configuration
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"openmetadata.org"}
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
# JWT Configuration
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
# OpenMetadata Server Pipeline Service Client Configuration
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
#Database configuration for postgresql
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
DB_SCHEME: ${DB_SCHEME:-postgresql}
DB_USE_SSL: ${DB_USE_SSL:-false}
DB_USER: ${DB_USER:-openmetadata_user}
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
DB_HOST: ${DB_HOST:-postgresql}
DB_PORT: ${DB_PORT:-5432}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10}
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
#eventMonitoringConfiguration
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
#pipelineServiceClientConfiguration
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
#airflow parameters
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
#secretsManagerConfiguration
SECRET_MANAGER: ${SECRET_MANAGER:-noop}
#parameters:
OM_SM_REGION: ${OM_SM_REGION:-""}
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
#email configuration:
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
#changeEventConfig
OM_URI: ${OM_URI:- "http://localhost:8585"}
#extensionConfiguration
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
# Heap OPTS Configurations
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
# Application Config
OM_CUSTOM_LOGO_URL_PATH: ${OM_CUSTOM_LOGO_URL_PATH:-""}
OM_CUSTOM_MONOGRAM_URL_PATH: ${OM_CUSTOM_MONOGRAM_URL_PATH:-""}
OM_MAX_FAILED_LOGIN_ATTEMPTS: ${OM_MAX_FAILED_LOGIN_ATTEMPTS:-3}
OM_LOGIN_ACCESS_BLOCK_TIME: ${OM_LOGIN_ACCESS_BLOCK_TIME:-600}
OM_JWT_EXPIRY_TIME: ${OM_JWT_EXPIRY_TIME:-3600}
# Mask passwords values in UI
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
#OpenMetadata Web Configuration
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
#HSTS
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
#Frame Options
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
#Content Type
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
#XSS-Protection
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
#CSP
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
depends_on:
elasticsearch:
condition: service_started
postgresql:
condition: service_healthy
networks:
- local_app_net
openmetadata-server:
build:
context: ../../.
@ -190,6 +344,8 @@ services:
condition: service_started
postgresql:
condition: service_healthy
execute-migrate-all:
condition: service_completed_successfully
networks:
- local_app_net
healthcheck:

157
docker/development/docker-compose.yml
View File

@ -57,6 +57,161 @@ services:
- "9300:9300"
volumes:
- es-data:/usr/share/elasticsearch/data
execute-migrate-all:
build:
context: ../../.
dockerfile: docker/development/Dockerfile
container_name: execute_migrate_all
command: "./bootstrap/bootstrap_storage.sh migrate-all"
environment:
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
SERVER_PORT: ${SERVER_PORT:-8585}
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
OPENMETADATA_DEBUG: ${OPENMETADATA_DEBUG:-false}
# Migration
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
# OpenMetadata Server Authentication Configuration
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"openmetadata.org"}
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
# JWT Configuration
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
# OpenMetadata Server Pipeline Service Client Configuration
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
# Database configuration for MySQL
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
DB_SCHEME: ${DB_SCHEME:-mysql}
DB_USE_SSL: ${DB_USE_SSL:-false}
DB_USER: ${DB_USER:-openmetadata_user}
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
DB_HOST: ${DB_HOST:-mysql}
DB_PORT: ${DB_PORT:-3306}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10}
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
#eventMonitoringConfiguration
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
#pipelineServiceClientConfiguration
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
#airflow parameters
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
#secretsManagerConfiguration
SECRET_MANAGER: ${SECRET_MANAGER:-noop}
#parameters:
OM_SM_REGION: ${OM_SM_REGION:-""}
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
#email configuration:
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
#changeEventConfig
OM_URI: ${OM_URI:- "http://localhost:8585"}
#extensionConfiguration
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
# Heap OPTS Configurations
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
# Application Config
OM_CUSTOM_LOGO_URL_PATH: ${OM_CUSTOM_LOGO_URL_PATH:-""}
OM_CUSTOM_MONOGRAM_URL_PATH: ${OM_CUSTOM_MONOGRAM_URL_PATH:-""}
OM_MAX_FAILED_LOGIN_ATTEMPTS: ${OM_MAX_FAILED_LOGIN_ATTEMPTS:-3}
OM_LOGIN_ACCESS_BLOCK_TIME: ${OM_LOGIN_ACCESS_BLOCK_TIME:-600}
OM_JWT_EXPIRY_TIME: ${OM_JWT_EXPIRY_TIME:-3600}
# Mask passwords values in UI
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
#OpenMetadata Web Configuration
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
#HSTS
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
#Frame Options
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
#Content Type
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
#XSS-Protection
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
#CSP
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
depends_on:
elasticsearch:
condition: service_started
mysql:
condition: service_healthy
networks:
- local_app_net
openmetadata-server:
build:
@ -190,6 +345,8 @@ services:
condition: service_started
mysql:
condition: service_healthy
execute-migrate-all:
condition: service_completed_successfully
networks:
- local_app_net
healthcheck:

153
docker/docker-compose-openmetadata/docker-compose-openmetadata.yml
View File

@ -11,6 +11,159 @@
version: "3.9"
services:
execute-migrate-all:
container_name: execute_migrate_all
command: "./bootstrap/bootstrap_storage.sh migrate-all"
image: docker.getcollate.io/openmetadata/server:1.2.0
environment:
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
SERVER_PORT: ${SERVER_PORT:-8585}
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
OPENMETADATA_DEBUG: ${OPENMETADATA_DEBUG:-false}
# Migration
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
# OpenMetadata Server Authentication Configuration
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"openmetadata.org"}
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
# JWT Configuration
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
# OpenMetadata Server Pipeline Service Client Configuration
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
# Database configuration for MySQL
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
DB_SCHEME: ${DB_SCHEME:-mysql}
DB_USE_SSL: ${DB_USE_SSL:-false}
DB_USER: ${DB_USER:-openmetadata_user}
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
DB_HOST: ${DB_HOST:-mysql}
DB_PORT: ${DB_PORT:-3306}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10}
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
#eventMonitoringConfiguration
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
#pipelineServiceClientConfiguration
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
#airflow parameters
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
#secretsManagerConfiguration
SECRET_MANAGER: ${SECRET_MANAGER:-noop}
#parameters:
OM_SM_REGION: ${OM_SM_REGION:-""}
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
#email configuration:
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
#changeEventConfig
OM_URI: ${OM_URI:- "http://localhost:8585"}
#extensionConfiguration
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
# Heap OPTS Configurations
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
# Application Config
OM_CUSTOM_LOGO_URL_PATH: ${OM_CUSTOM_LOGO_URL_PATH:-""}
OM_CUSTOM_MONOGRAM_URL_PATH: ${OM_CUSTOM_MONOGRAM_URL_PATH:-""}
OM_MAX_FAILED_LOGIN_ATTEMPTS: ${OM_MAX_FAILED_LOGIN_ATTEMPTS:-3}
OM_LOGIN_ACCESS_BLOCK_TIME: ${OM_LOGIN_ACCESS_BLOCK_TIME:-600}
OM_JWT_EXPIRY_TIME: ${OM_JWT_EXPIRY_TIME:-3600}
# Mask passwords values in UI
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
#OpenMetadata Web Configuration
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
#HSTS
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
#Frame Options
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
#Content Type
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
#XSS-Protection
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
#CSP
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
depends_on:
elasticsearch:
condition: service_started
mysql:
condition: service_healthy
networks:
- app_net
openmetadata-server:
container_name: openmetadata_server
restart: always

4
docker/docker-compose-openmetadata/env-mysql
View File

@ -3,6 +3,10 @@ SERVER_PORT="8585"
SERVER_ADMIN_PORT="8586"
LOG_LEVEL="INFO"
OPENMETADATA_DEBUG="false"
# Migration
MIGRATION_LIMIT_PARAM = 1200
# OpenMetadata Server Authentication Configuration
AUTHORIZER_CLASS_NAME="org.openmetadata.service.security.DefaultAuthorizer"
AUTHORIZER_REQUEST_FILTER="org.openmetadata.service.security.JwtFilter"

4
docker/docker-compose-openmetadata/env-postgres
View File

@ -3,6 +3,10 @@ SERVER_PORT="8585"
SERVER_ADMIN_PORT="8586"
LOG_LEVEL="INFO"
OPENMETADATA_DEBUG="false"
# Migration
MIGRATION_LIMIT_PARAM = 1200
# OpenMetadata Server Authentication Configuration
AUTHORIZER_CLASS_NAME="org.openmetadata.service.security.DefaultAuthorizer"
AUTHORIZER_REQUEST_FILTER="org.openmetadata.service.security.JwtFilter"

158
docker/docker-compose-quickstart/docker-compose-postgres.yml
View File

@ -53,10 +53,162 @@ services:
volumes:
- es-data:/usr/share/elasticsearch/data
execute-migrate-all:
container_name: execute_migrate_all
image: docker.getcollate.io/openmetadata/server:1.1.0
command: "./bootstrap/bootstrap_storage.sh migrate-all"
environment:
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
SERVER_PORT: ${SERVER_PORT:-8585}
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
OPENMETADATA_DEBUG: ${OPENMETADATA_DEBUG:-false}
# Migration
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
# OpenMetadata Server Authentication Configuration
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"openmetadata.org"}
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
# JWT Configuration
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
# OpenMetadata Server Pipeline Service Client Configuration
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
#Database configuration for postgresql
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-org.postgresql.Driver}
DB_SCHEME: ${DB_SCHEME:-postgresql}
DB_USE_SSL: ${DB_USE_SSL:-false}
DB_USER: ${DB_USER:-openmetadata_user}
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
DB_HOST: ${DB_HOST:-postgresql}
DB_PORT: ${DB_PORT:-5432}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10}
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
#eventMonitoringConfiguration
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
#pipelineServiceClientConfiguration
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
#airflow parameters
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
#secretsManagerConfiguration
SECRET_MANAGER: ${SECRET_MANAGER:-noop}
#parameters:
OM_SM_REGION: ${OM_SM_REGION:-""}
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
#email configuration:
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
#changeEventConfig
OM_URI: ${OM_URI:- "http://localhost:8585"}
#extensionConfiguration
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
# Heap OPTS Configurations
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
# Application Config
OM_CUSTOM_LOGO_URL_PATH: ${OM_CUSTOM_LOGO_URL_PATH:-""}
OM_CUSTOM_MONOGRAM_URL_PATH: ${OM_CUSTOM_MONOGRAM_URL_PATH:-""}
OM_MAX_FAILED_LOGIN_ATTEMPTS: ${OM_MAX_FAILED_LOGIN_ATTEMPTS:-3}
OM_LOGIN_ACCESS_BLOCK_TIME: ${OM_LOGIN_ACCESS_BLOCK_TIME:-600}
OM_JWT_EXPIRY_TIME: ${OM_JWT_EXPIRY_TIME:-3600}
# Mask passwords values in UI
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
#OpenMetadata Web Configuration
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
#HSTS
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
#Frame Options
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
#Content Type
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
#XSS-Protection
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
#CSP
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
depends_on:
elasticsearch:
condition: service_started
postgresql:
condition: service_healthy
networks:
- app_net
openmetadata-server:
container_name: openmetadata_server
restart: always
image: docker.getcollate.io/openmetadata/server:1.0.1
image: docker.getcollate.io/openmetadata/server:1.1.0
environment:
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
SERVER_PORT: ${SERVER_PORT:-8585}
@ -206,6 +358,8 @@ services:
condition: service_started
postgresql:
condition: service_healthy
execute-migrate-all:
condition: service_completed_successfully
networks:
- app_net
healthcheck:
@ -213,7 +367,7 @@ services:
ingestion:
container_name: openmetadata_ingestion
image: docker.getcollate.io/openmetadata/ingestion:1.0.1
image: docker.getcollate.io/openmetadata/ingestion:1.1.0
depends_on:
elasticsearch:
condition: service_started

157
docker/docker-compose-quickstart/docker-compose.yml
View File

@ -51,10 +51,163 @@ services:
volumes:
- es-data:/usr/share/elasticsearch/data
execute-migrate-all:
container_name: execute_migrate_all
image: docker.getcollate.io/openmetadata/server:1.1.0
command: "./bootstrap/bootstrap_storage.sh migrate-all"
environment:
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
SERVER_PORT: ${SERVER_PORT:-8585}
SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
OPENMETADATA_DEBUG: ${OPENMETADATA_DEBUG:-false}
# Migration
MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}
# OpenMetadata Server Authentication Configuration
AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"openmetadata.org"}
AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
# JWT Configuration
RSA_PUBLIC_KEY_FILE_PATH: ${RSA_PUBLIC_KEY_FILE_PATH:-"./conf/public_key.der"}
RSA_PRIVATE_KEY_FILE_PATH: ${RSA_PRIVATE_KEY_FILE_PATH:-"./conf/private_key.der"}
JWT_ISSUER: ${JWT_ISSUER:-"open-metadata.org"}
JWT_KEY_ID: ${JWT_KEY_ID:-"Gb389a-9f76-gdjs-a92j-0242bk94356"}
# OpenMetadata Server Pipeline Service Client Configuration
PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}
SERVER_HOST_API_URL: ${SERVER_HOST_API_URL:-http://openmetadata-server:8585/api}
PIPELINE_SERVICE_CLIENT_VERIFY_SSL: ${PIPELINE_SERVICE_CLIENT_VERIFY_SSL:-"no-ssl"}
PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH: ${PIPELINE_SERVICE_CLIENT_SSL_CERT_PATH:-""}
# Database configuration for MySQL
DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
DB_SCHEME: ${DB_SCHEME:-mysql}
DB_USE_SSL: ${DB_USE_SSL:-false}
DB_USER: ${DB_USER:-openmetadata_user}
DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
DB_HOST: ${DB_HOST:-mysql}
DB_PORT: ${DB_PORT:-3306}
OM_DATABASE: ${OM_DATABASE:-openmetadata_db}
# ElasticSearch Configurations
ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:- elasticsearch}
ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
SEARCH_TYPE: ${SEARCH_TYPE:- "elasticsearch"}
ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-5}
ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS: ${ELASTICSEARCH_KEEP_ALIVE_TIMEOUT_SECS:-600}
ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10}
ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
#eventMonitoringConfiguration
EVENT_MONITOR: ${EVENT_MONITOR:-prometheus}
EVENT_MONITOR_BATCH_SIZE: ${EVENT_MONITOR_BATCH_SIZE:-10}
EVENT_MONITOR_PATH_PATTERN: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
EVENT_MONITOR_LATENCY: ${EVENT_MONITOR_LATENCY:-[]}
#pipelineServiceClientConfiguration
PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
#airflow parameters
AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}
#secretsManagerConfiguration
SECRET_MANAGER: ${SECRET_MANAGER:-noop}
#parameters:
OM_SM_REGION: ${OM_SM_REGION:-""}
OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
#email configuration:
OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}
#changeEventConfig
OM_URI: ${OM_URI:- "http://localhost:8585"}
#extensionConfiguration
OM_RESOURCE_PACKAGES: ${OM_RESOURCE_PACKAGES:-[]}
OM_EXTENSIONS: ${OM_EXTENSIONS:-[]}
# Heap OPTS Configurations
OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx1G -Xms1G}
# Application Config
OM_CUSTOM_LOGO_URL_PATH: ${OM_CUSTOM_LOGO_URL_PATH:-""}
OM_CUSTOM_MONOGRAM_URL_PATH: ${OM_CUSTOM_MONOGRAM_URL_PATH:-""}
OM_MAX_FAILED_LOGIN_ATTEMPTS: ${OM_MAX_FAILED_LOGIN_ATTEMPTS:-3}
OM_LOGIN_ACCESS_BLOCK_TIME: ${OM_LOGIN_ACCESS_BLOCK_TIME:-600}
OM_JWT_EXPIRY_TIME: ${OM_JWT_EXPIRY_TIME:-3600}
# Mask passwords values in UI
MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}
#OpenMetadata Web Configuration
WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
#HSTS
WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
#Frame Options
WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
#Content Type
WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
#XSS-Protection
WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
#CSP
WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
depends_on:
elasticsearch:
condition: service_started
mysql:
condition: service_healthy
networks:
- app_net
openmetadata-server:
container_name: openmetadata_server
restart: always
image: docker.getcollate.io/openmetadata/server:1.0.1
image: docker.getcollate.io/openmetadata/server:1.1.0
environment:
OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
SERVER_PORT: ${SERVER_PORT:-8585}
@ -212,7 +365,7 @@ services:
ingestion:
container_name: openmetadata_ingestion
image: docker.getcollate.io/openmetadata/ingestion:1.0.1
image: docker.getcollate.io/openmetadata/ingestion:1.1.0
depends_on:
elasticsearch:
condition: service_started

4
docker/openmetadata-start.sh
View File

@ -11,8 +11,8 @@
# limitations under the License.
echo "Initializing OpenMetadata Server...";
echo "Migrating the database to the latest version and the indexes in ElasticSearch...";
./bootstrap/bootstrap_storage.sh migrate-all
# echo "Migrating the database to the latest version and the indexes in ElasticSearch...";
# ./bootstrap/bootstrap_storage.sh migrate-all
echo " ||||||| "
echo " |||| |||| ____ "
echo " |||| |||| / __ \ "