OpenMetadata/docker/docker-compose-quickstart/docker-compose-rdf.yml

#  Copyright 2021 Collate
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  http://www.apache.org/licenses/LICENSE-2.0
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

# OpenMetadata Quickstart with RDF/Knowledge Graph support
# Usage: docker-compose -f docker-compose-rdf.yml up

version: "3.9"
volumes:
  ingestion-volume-dag-airflow:
  ingestion-volume-dags:
  ingestion-volume-tmp:
  es-data:
  fuseki-data:

services:
  mysql:
    container_name: openmetadata_mysql
    image: docker.getcollate.io/openmetadata/db:1.8.0-SNAPSHOT
    command: "--sort_buffer_size=10M"
    restart: always
    environment:
      MYSQL_ROOT_PASSWORD: password
    expose:
      - 3306
    ports:
      - "3306:3306"
    volumes:
     - ./docker-volume/db-data:/var/lib/mysql
    networks:
      - app_net
    healthcheck:
      test: mysql --user=root --password=$$MYSQL_ROOT_PASSWORD --silent --execute "use openmetadata_db"
      interval: 15s
      timeout: 10s
      retries: 10

  elasticsearch:
    container_name: openmetadata_elasticsearch
    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
    environment:
      - discovery.type=single-node
      - ES_JAVA_OPTS=-Xms2g -Xmx4g
      - xpack.security.enabled=false
      - indices.memory.index_buffer_size=20%
      - cluster.routing.allocation.disk.threshold_enabled=false
    networks:
      - app_net
    ports:
      - "9200:9200"
      - "9300:9300"
    healthcheck:
      test: "curl -s http://localhost:9200/_cluster/health?pretty | grep status | grep -qE 'green|yellow' || exit 1"
      interval: 15s
      timeout: 10s
      retries: 10
    volumes:
      - es-data:/usr/share/elasticsearch/data
    deploy:
      resources:
        limits:
          memory: 4G
        reservations:
          memory: 2G

  # Apache Jena Fuseki for RDF/Knowledge Graph storage
  fuseki:
    container_name: openmetadata_fuseki
    image: stain/jena-fuseki:4.10.0
    restart: always
    environment:
      - ADMIN_PASSWORD=${FUSEKI_ADMIN_PASSWORD:-admin}
      - FUSEKI_DATASET_1=openmetadata
      - JVM_ARGS=-Xmx4g -Xms2g
      - FUSEKI_BASE=/fuseki
    ports:
      - "3030:3030"
    volumes:
      - fuseki-data:/fuseki
    networks:
      - app_net
    healthcheck:
      test: ["CMD", "wget", "-q", "--spider", "http://localhost:3030/$/ping"]
      interval: 15s
      timeout: 10s
      retries: 10
      start_period: 40s
    deploy:
      resources:
        limits:
          memory: 4G
        reservations:
          memory: 2G

  execute-migrate-all:
    container_name: execute_migrate_all
    image: docker.getcollate.io/openmetadata/server:1.8.0-SNAPSHOT
    command: "./bootstrap/openmetadata-ops.sh migrate"
    environment:
      OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
      SERVER_PORT: ${SERVER_PORT:-8585}
      SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
      LOG_LEVEL: ${LOG_LEVEL:-INFO}
      
      # Migration 
      MIGRATION_LIMIT_PARAM: ${MIGRATION_LIMIT_PARAM:-1200}

      # OpenMetadata Server Authentication Configuration
      AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
      AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
      AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
      AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
      AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
      AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
      AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
      AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
      AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
      AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
      AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
      CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
      AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
      AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
      AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
      AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
      AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
      AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
      AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
      AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
      #For OIDC Authentication, when client is confidential
      OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
      OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
      OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
      OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
      OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
      OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
      OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-RS256}
      OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
      OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
      OIDC_CALLBACK: ${OIDC_CALLBACK:-""}
      OIDC_SERVER_URL: ${OIDC_SERVER_URL:-""}
      OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
      OIDC_TENANT: ${OIDC_TENANT:-""}
      OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
      OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}

      # Database configuration for MySQL
      DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
      DB_SCHEME: ${DB_SCHEME:-mysql}
      DB_USE_SSL: ${DB_USE_SSL:-false}
      DB_USER: ${DB_USER:-openmetadata_user}
      DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
      DB_HOST: ${DB_HOST:-mysql}
      DB_PORT: ${DB_PORT:-3306}
      OM_DATABASE: ${OM_DATABASE:-openmetadata_db}

      # Elasticsearch configuration
      ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
      ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-20}
      ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-elasticsearch}
      ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
      ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
      ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
      ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
      SEARCH_TYPE: ${SEARCH_TYPE:-elasticsearch}
      ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
      ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
      ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
      ELASTICSEARCH_CA_CERT_PATH: ${ELASTICSEARCH_CA_CERT_PATH:-""}
      ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10}
      ELASTICSEARCH_PAYLOAD_SIZE_IN_BYTES: ${ELASTICSEARCH_PAYLOAD_SIZE_IN_BYTES:-10485760}
      ELASTICSEARCH_INDEX_FACTORY: ${ELASTICSEARCH_INDEX_FACTORY:-"org.openmetadata.service.search.elasticsearch.ElasticSearchIndexFactory"}

      # RDF Configuration
      RDF_ENABLED: ${RDF_ENABLED:-true}
      RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
      RDF_REMOTE_ENDPOINT: ${RDF_REMOTE_ENDPOINT:-http://fuseki:3030/openmetadata}
      RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
      RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-${FUSEKI_ADMIN_PASSWORD:-admin}}
      RDF_BASE_URI: ${RDF_BASE_URI:-https://open-metadata.org/}
      RDF_JSONLD_ENABLED: ${RDF_JSONLD_ENABLED:-true}
      RDF_SPARQL_ENABLED: ${RDF_SPARQL_ENABLED:-true}
      RDF_AUTO_GENERATE: ${RDF_AUTO_GENERATE:-true}
      RDF_SYNC_BATCH_SIZE: ${RDF_SYNC_BATCH_SIZE:-100}

      # Pipeline Service Client Configuration
      PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
      PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
      PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
      PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
      PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
      #airflow parameters
      AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
      AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
      AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
      AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
      AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
      FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}

      #secretsManagerConfiguration
      SECRET_MANAGER: ${SECRET_MANAGER:-db}
      # AWS:
      OM_SM_REGION: ${OM_SM_REGION:-""}
      OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
      OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
      # Azure:
      OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
      OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
      OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
      OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}

      #email configuration:
      OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
      OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
      AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
      OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
      OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
      SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
      SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
      SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
      SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
      SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}

      # Heap OPTS Configurations
      OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx2G -Xms1G}
      # Mask passwords values in UI
      MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}

      #OpenMetadata Web Configuration
      WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
      #HSTS
      WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
      WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
      WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
      WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
      #Frame Options
      WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
      WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
      WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
      #Content Type
      WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
      #XSS-Protection  
      WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
      WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
      WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
      #CSP    
      WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
      WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
      WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
      #Referrer-Policy
      WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
      WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
      #Permission-Policy
      WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
      WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
      #Cache
      WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
      WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
    depends_on:
      elasticsearch:
        condition: service_healthy
      mysql:
        condition: service_healthy
      fuseki:
        condition: service_healthy
    networks:
      - app_net

  openmetadata-server:
    container_name: openmetadata_server
    restart: always
    image: docker.getcollate.io/openmetadata/server:1.8.0-SNAPSHOT
    environment:
      OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
      SERVER_PORT: ${SERVER_PORT:-8585}
      SERVER_ADMIN_PORT: ${SERVER_ADMIN_PORT:-8586}
      LOG_LEVEL: ${LOG_LEVEL:-INFO}

      # OpenMetadata Server Authentication Configuration
      AUTHORIZER_CLASS_NAME: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
      AUTHORIZER_REQUEST_FILTER: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
      AUTHORIZER_ADMIN_PRINCIPALS: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
      AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN: ${AUTHORIZER_ALLOWED_REGISTRATION_DOMAIN:-["all"]}
      AUTHORIZER_INGESTION_PRINCIPALS: ${AUTHORIZER_INGESTION_PRINCIPALS:-[ingestion-bot]}
      AUTHORIZER_PRINCIPAL_DOMAIN: ${AUTHORIZER_PRINCIPAL_DOMAIN:-"open-metadata.org"}
      AUTHORIZER_ALLOWED_DOMAINS: ${AUTHORIZER_ALLOWED_DOMAINS:-[]}
      AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN: ${AUTHORIZER_ENFORCE_PRINCIPAL_DOMAIN:-false}
      AUTHORIZER_ENABLE_SECURE_SOCKET: ${AUTHORIZER_ENABLE_SECURE_SOCKET:-false}
      AUTHENTICATION_PROVIDER: ${AUTHENTICATION_PROVIDER:-basic}
      AUTHENTICATION_RESPONSE_TYPE: ${AUTHENTICATION_RESPONSE_TYPE:-id_token}
      CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
      AUTHENTICATION_PUBLIC_KEYS: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
      AUTHENTICATION_AUTHORITY: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
      AUTHENTICATION_CLIENT_ID: ${AUTHENTICATION_CLIENT_ID:-""}
      AUTHENTICATION_CALLBACK_URL: ${AUTHENTICATION_CALLBACK_URL:-""}
      AUTHENTICATION_JWT_PRINCIPAL_CLAIMS: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS:-[email,preferred_username,sub]}
      AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING:-[]}
      AUTHENTICATION_ENABLE_SELF_SIGNUP: ${AUTHENTICATION_ENABLE_SELF_SIGNUP:-true}
      AUTHENTICATION_CLIENT_TYPE: ${AUTHENTICATION_CLIENT_TYPE:-public}
      #For OIDC Authentication, when client is confidential
      OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-""}
      OIDC_TYPE: ${OIDC_TYPE:-""} # google, azure etc.
      OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-""}
      OIDC_SCOPE: ${OIDC_SCOPE:-"openid email profile"}
      OIDC_DISCOVERY_URI: ${OIDC_DISCOVERY_URI:-""}
      OIDC_USE_NONCE: ${OIDC_USE_NONCE:-true}
      OIDC_PREFERRED_JWS: ${OIDC_PREFERRED_JWS:-RS256}
      OIDC_RESPONSE_TYPE: ${OIDC_RESPONSE_TYPE:-"code"}
      OIDC_DISABLE_PKCE: ${OIDC_DISABLE_PKCE:-true}
      OIDC_CALLBACK: ${OIDC_CALLBACK:-""}
      OIDC_SERVER_URL: ${OIDC_SERVER_URL:-""}
      OIDC_CLIENT_AUTH_METHOD: ${OIDC_CLIENT_AUTH_METHOD:-"client_secret_post"}
      OIDC_TENANT: ${OIDC_TENANT:-""}
      OIDC_MAX_CLOCK_SKEW: ${OIDC_MAX_CLOCK_SKEW:-""}
      OIDC_CUSTOM_PARAMS: ${OIDC_CUSTOM_PARAMS:-{}}

      # Database configuration for MySQL
      DB_DRIVER_CLASS: ${DB_DRIVER_CLASS:-com.mysql.cj.jdbc.Driver}
      DB_SCHEME: ${DB_SCHEME:-mysql}
      DB_USE_SSL: ${DB_USE_SSL:-false}
      DB_USER: ${DB_USER:-openmetadata_user}
      DB_USER_PASSWORD: ${DB_USER_PASSWORD:-openmetadata_password}
      DB_HOST: ${DB_HOST:-mysql}
      DB_PORT: ${DB_PORT:-3306}
      OM_DATABASE: ${OM_DATABASE:-openmetadata_db}

      # Elasticsearch configuration
      ELASTICSEARCH_SOCKET_TIMEOUT_SECS: ${ELASTICSEARCH_SOCKET_TIMEOUT_SECS:-60}
      ELASTICSEARCH_CONNECTION_TIMEOUT_SECS: ${ELASTICSEARCH_CONNECTION_TIMEOUT_SECS:-20}
      ELASTICSEARCH_HOST: ${ELASTICSEARCH_HOST:-elasticsearch}
      ELASTICSEARCH_PORT: ${ELASTICSEARCH_PORT:-9200}
      ELASTICSEARCH_SCHEME: ${ELASTICSEARCH_SCHEME:-http}
      ELASTICSEARCH_USER: ${ELASTICSEARCH_USER:-""}
      ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-""}
      SEARCH_TYPE: ${SEARCH_TYPE:-elasticsearch}
      ELASTICSEARCH_TRUST_STORE_PATH: ${ELASTICSEARCH_TRUST_STORE_PATH:-""}
      ELASTICSEARCH_TRUST_STORE_PASSWORD: ${ELASTICSEARCH_TRUST_STORE_PASSWORD:-""}
      ELASTICSEARCH_INDEX_MAPPING_LANG: ${ELASTICSEARCH_INDEX_MAPPING_LANG:-EN}
      ELASTICSEARCH_CA_CERT_PATH: ${ELASTICSEARCH_CA_CERT_PATH:-""}
      ELASTICSEARCH_BATCH_SIZE: ${ELASTICSEARCH_BATCH_SIZE:-10}
      ELASTICSEARCH_PAYLOAD_SIZE_IN_BYTES: ${ELASTICSEARCH_PAYLOAD_SIZE_IN_BYTES:-10485760}
      ELASTICSEARCH_INDEX_FACTORY: ${ELASTICSEARCH_INDEX_FACTORY:-"org.openmetadata.service.search.elasticsearch.ElasticSearchIndexFactory"}

      # RDF Configuration
      RDF_ENABLED: ${RDF_ENABLED:-true}
      RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
      RDF_REMOTE_ENDPOINT: ${RDF_REMOTE_ENDPOINT:-http://fuseki:3030/openmetadata}
      RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
      RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-${FUSEKI_ADMIN_PASSWORD:-admin}}
      RDF_BASE_URI: ${RDF_BASE_URI:-https://open-metadata.org/}
      RDF_JSONLD_ENABLED: ${RDF_JSONLD_ENABLED:-true}
      RDF_SPARQL_ENABLED: ${RDF_SPARQL_ENABLED:-true}
      RDF_AUTO_GENERATE: ${RDF_AUTO_GENERATE:-true}
      RDF_SYNC_BATCH_SIZE: ${RDF_SYNC_BATCH_SIZE:-100}

      # Pipeline Service Client Configuration
      PIPELINE_SERVICE_CLIENT_ENDPOINT: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://ingestion:8080}
      PIPELINE_SERVICE_CLIENT_CLASS_NAME: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
      PIPELINE_SERVICE_IP_INFO_ENABLED: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
      PIPELINE_SERVICE_CLIENT_HOST_IP: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
      PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER: ${PIPELINE_SERVICE_CLIENT_SECRETS_MANAGER_LOADER:-"noop"}
      #airflow parameters
      AIRFLOW_USERNAME: ${AIRFLOW_USERNAME:-admin}
      AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD:-admin}
      AIRFLOW_TIMEOUT: ${AIRFLOW_TIMEOUT:-10}
      AIRFLOW_TRUST_STORE_PATH: ${AIRFLOW_TRUST_STORE_PATH:-""}
      AIRFLOW_TRUST_STORE_PASSWORD: ${AIRFLOW_TRUST_STORE_PASSWORD:-""}
      FERNET_KEY: ${FERNET_KEY:-jJ/9sz0g0OHxsfxOoSfdFdmk3ysNmPRnH3TUAbz3IHA=}

      #secretsManagerConfiguration
      SECRET_MANAGER: ${SECRET_MANAGER:-db}
      # AWS:
      OM_SM_REGION: ${OM_SM_REGION:-""}
      OM_SM_ACCESS_KEY_ID: ${OM_SM_ACCESS_KEY_ID:-""}
      OM_SM_ACCESS_KEY: ${OM_SM_ACCESS_KEY:-""}
      # Azure:
      OM_SM_VAULT_NAME: ${OM_SM_VAULT_NAME:-""}
      OM_SM_CLIENT_ID: ${OM_SM_CLIENT_ID:-""}
      OM_SM_CLIENT_SECRET: ${OM_SM_CLIENT_SECRET:-""}
      OM_SM_TENANT_ID: ${OM_SM_TENANT_ID:-""}

      #email configuration:
      OM_EMAIL_ENTITY: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
      OM_SUPPORT_URL: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
      AUTHORIZER_ENABLE_SMTP : ${AUTHORIZER_ENABLE_SMTP:-false}
      OPENMETADATA_SERVER_URL: ${OPENMETADATA_SERVER_URL:-""}
      OPENMETADATA_SMTP_SENDER_MAIL: ${OPENMETADATA_SMTP_SENDER_MAIL:-""}
      SMTP_SERVER_ENDPOINT: ${SMTP_SERVER_ENDPOINT:-""}
      SMTP_SERVER_PORT: ${SMTP_SERVER_PORT:-""}
      SMTP_SERVER_USERNAME: ${SMTP_SERVER_USERNAME:-""}
      SMTP_SERVER_PWD: ${SMTP_SERVER_PWD:-""}
      SMTP_SERVER_STRATEGY: ${SMTP_SERVER_STRATEGY:-"SMTP_TLS"}

      # Heap OPTS Configurations
      OPENMETADATA_HEAP_OPTS: ${OPENMETADATA_HEAP_OPTS:--Xmx2G -Xms1G}
      # Mask passwords values in UI
      MASK_PASSWORDS_API: ${MASK_PASSWORDS_API:-false}

      #OpenMetadata Web Configuration
      WEB_CONF_URI_PATH: ${WEB_CONF_URI_PATH:-"/api"}
      #HSTS
      WEB_CONF_HSTS_ENABLED: ${WEB_CONF_HSTS_ENABLED:-false}
      WEB_CONF_HSTS_MAX_AGE: ${WEB_CONF_HSTS_MAX_AGE:-"365 days"}
      WEB_CONF_HSTS_INCLUDE_SUBDOMAINS: ${WEB_CONF_HSTS_INCLUDE_SUBDOMAINS:-"true"}
      WEB_CONF_HSTS_PRELOAD: ${WEB_CONF_HSTS_PRELOAD:-"true"}
      #Frame Options
      WEB_CONF_FRAME_OPTION_ENABLED: ${WEB_CONF_FRAME_OPTION_ENABLED:-false}
      WEB_CONF_FRAME_OPTION: ${WEB_CONF_FRAME_OPTION:-"SAMEORIGIN"}
      WEB_CONF_FRAME_ORIGIN: ${WEB_CONF_FRAME_ORIGIN:-""}
      #Content Type
      WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED: ${WEB_CONF_CONTENT_TYPE_OPTIONS_ENABLED:-false}
      #XSS-Protection  
      WEB_CONF_XSS_PROTECTION_ENABLED: ${WEB_CONF_XSS_PROTECTION_ENABLED:-false}
      WEB_CONF_XSS_PROTECTION_ON: ${WEB_CONF_XSS_PROTECTION_ON:-true}
      WEB_CONF_XSS_PROTECTION_BLOCK: ${WEB_CONF_XSS_PROTECTION_BLOCK:-true}
      #CSP    
      WEB_CONF_XSS_CSP_ENABLED: ${WEB_CONF_XSS_CSP_ENABLED:-false}
      WEB_CONF_XSS_CSP_POLICY: ${WEB_CONF_XSS_CSP_POLICY:-"default-src 'self'"}
      WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY: ${WEB_CONF_XSS_CSP_REPORT_ONLY_POLICY:-""}
      #Referrer-Policy
      WEB_CONF_REFERRER_POLICY_ENABLED: ${WEB_CONF_REFERRER_POLICY_ENABLED:-false}
      WEB_CONF_REFERRER_POLICY_OPTION: ${WEB_CONF_REFERRER_POLICY_OPTION:-"SAME_ORIGIN"}
      #Permission-Policy
      WEB_CONF_PERMISSION_POLICY_ENABLED: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
      WEB_CONF_PERMISSION_POLICY_OPTION: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
      #Cache
      WEB_CONF_CACHE_CONTROL: ${WEB_CONF_CACHE_CONTROL:-""}
      WEB_CONF_PRAGMA: ${WEB_CONF_PRAGMA:-""}
    expose:
      - 8585
      - 8586
    ports:
      - "8585:8585"
      - "8586:8586"
    depends_on:
      execute-migrate-all:
        condition: service_completed_successfully
      fuseki:
        condition: service_healthy
    networks:
      - app_net
    healthcheck:
      test: [ "CMD", "wget", "-q", "--spider",  "http://localhost:8586/healthcheck" ]

  ingestion:
    container_name: openmetadata_ingestion
    build:
      context: ../
      dockerfile: docker/development/Dockerfile
    image: docker.getcollate.io/openmetadata/ingestion:1.8.0-SNAPSHOT
    depends_on:
      elasticsearch:
        condition: service_healthy
      mysql:
        condition: service_healthy
      openmetadata-server:
        condition: service_healthy
      fuseki:
        condition: service_healthy
    environment:
      AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__OPENMETADATA_AIRFLOW_APIS__DAG_GENERATED_CONFIGS: "/opt/airflow/dag_generated_configs"
      DB_SCHEME: ${AIRFLOW_DB_SCHEME:-mysql+pymysql}
      DB_HOST: ${AIRFLOW_DB_HOST:-mysql}
      DB_PORT: ${AIRFLOW_DB_PORT:-3306}
      AIRFLOW_DB: ${AIRFLOW_DB:-airflow_db}
      DB_USER: ${AIRFLOW_DB_USER:-airflow_user}
      DB_PASSWORD: ${AIRFLOW_DB_PASSWORD:-airflow_pass}

      # RDF Configuration (for any RDF-aware ingestion pipelines)
      RDF_ENABLED: ${RDF_ENABLED:-true}
      RDF_STORAGE_TYPE: ${RDF_STORAGE_TYPE:-FUSEKI}
      RDF_REMOTE_ENDPOINT: ${RDF_REMOTE_ENDPOINT:-http://fuseki:3030/openmetadata}
      RDF_REMOTE_USERNAME: ${RDF_REMOTE_USERNAME:-admin}
      RDF_REMOTE_PASSWORD: ${RDF_REMOTE_PASSWORD:-${FUSEKI_ADMIN_PASSWORD:-admin}}
    entrypoint: /bin/bash
    command:
      - "/opt/airflow/ingestion_dependency.sh"
    expose:
      - 8080
    ports:
      - "8080:8080"
    networks:
      - app_net
    volumes:
      - ingestion-volume-dag-airflow:/opt/airflow/dag_generated_configs
      - ingestion-volume-dags:/opt/airflow/dags
      - ingestion-volume-tmp:/tmp

networks:
  app_net:
    driver: bridge