From eda67ccdffbccd80f88f00a7ef48adde2471cc9f Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 12 Aug 2021 12:24:29 -0700 Subject: [PATCH] Update setup, ingestion docs --- docs/SUMMARY.md | 26 +-- docs/install/metadata-ingestion/README.md | 2 +- .../openmetadata-configuration.md | 161 ++++++++++++++++++ docs/install/run-openmetadata.md | 2 +- 4 files changed, 177 insertions(+), 14 deletions(-) create mode 100644 docs/install/metadata-ingestion/openmetadata-configuration.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 5ebe3a62153..8c21ff6a506 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -38,7 +38,20 @@ ## Install * [Run OpenMetadata](install/run-openmetadata.md) -* [Run in production](install/run-in-production.md) +* [Run in Production](install/run-in-production.md) +* [OpenMetadata Configuration](install/openmetadata-configuration.md) +* [Enable Security](install/enable-security/README.md) + * [Google SSO](install/enable-security/google-sso/README.md) + * [Create Server Credentials](install/enable-security/google-sso/google-server-creds.md) + * [Create Service Account](install/enable-security/google-sso/create-ingestion-service-account.md) + * [Configure OpenMetadata Server](install/enable-security/google-sso/google-catalog-config.md) + * [Configure Ingestion](install/enable-security/google-sso/configure-security-ingestion.md) + * [Okta SSO](install/enable-security/okta-sso/README.md) + * [Create Server Credentials](install/enable-security/okta-sso/okta-server-creds.md) + * [Configure OpenMetadata Server](install/enable-security/okta-sso/okta-catalog-config.md) + * [Create Service Account](install/enable-security/okta-sso/create-ingestion-service-account.md) + * [Configure Ingestion](install/enable-security/okta-sso/configure-security-ingestion.md) + * [Metadata Ingestion](install/metadata-ingestion/README.md) * [Ingest Sample Data](install/metadata-ingestion/ingest-sample-data.md) * [Connectors](install/metadata-ingestion/connectors/README.md) @@ -55,17 +68,6 @@ * [ElasticSearch](install/metadata-ingestion/connectors/elastic-search.md) * [Scheduler](install/metadata-ingestion/scheduler.md) * [Airflow](install/metadata-ingestion/airflow.md) -* [Enable Security](install/enable-security/README.md) - * [Google SSO](install/enable-security/google-sso/README.md) - * [Create Server Credentials](install/enable-security/google-sso/google-server-creds.md) - * [Create Service Account](install/enable-security/google-sso/create-ingestion-service-account.md) - * [Configure OpenMetadata Server](install/enable-security/google-sso/google-catalog-config.md) - * [Configure Ingestion](install/enable-security/google-sso/configure-security-ingestion.md) - * [Okta SSO](install/enable-security/okta-sso/README.md) - * [Create Server Credentials](install/enable-security/okta-sso/okta-server-creds.md) - * [Configure OpenMetadata Server](install/enable-security/okta-sso/okta-catalog-config.md) - * [Create Service Account](install/enable-security/okta-sso/create-ingestion-service-account.md) - * [Configure Ingestion](install/enable-security/okta-sso/configure-security-ingestion.md) ## Open Source Community diff --git a/docs/install/metadata-ingestion/README.md b/docs/install/metadata-ingestion/README.md index a5c7d45c07a..66ff2f4903c 100644 --- a/docs/install/metadata-ingestion/README.md +++ b/docs/install/metadata-ingestion/README.md @@ -1,6 +1,6 @@ # Metadata Ingestion -OpenMetadata Ingesiton is a simple framework to build connectors and ingest metadata of various systems through OpenMetadata APIs. +OpenMetadata Ingesiton is a simple framework to build connectors and ingest metadata of various systems through [OpenMetadata APIs](https://docs.open-metadata.org/v/docs/openmetadata-apis/apis). It could be used in an orchestration framework(e.g. Apache Airflow) to ingest metadata. ## Ingest Sample Data diff --git a/docs/install/metadata-ingestion/openmetadata-configuration.md b/docs/install/metadata-ingestion/openmetadata-configuration.md new file mode 100644 index 00000000000..0e9f1d930ce --- /dev/null +++ b/docs/install/metadata-ingestion/openmetadata-configuration.md @@ -0,0 +1,161 @@ +--- +description: >- + This document describes OpenMetadata Server Configuration +--- + +# OpenMetadata Server Configuration + +```text +swagger: + resourcePackage: org.openmetadata.catalog.resources + + +server: + rootPath: '/api/*' + applicationConnectors: + - type: http + port: 8585 + adminConnectors: + - type: http + port: 8586 + +# Logging settings. +# https://logback.qos.ch/manual/layouts.html#conversionWord +logging: + level: INFO + loggers: + org.openmetadata.catalog.common: DEBUG + io.swagger: ERROR + appenders: + - type: file + threshold: TRACE + logFormat: "%level [%d{HH:mm:ss.SSS}] [%t] %logger{5} - %msg %n" + currentLogFilename: ./logs/openmetadata.log + archivedLogFilenamePattern: ./logs/openmetadata-%d{yyyy-MM-dd}-%i.log.gz + archivedFileCount: 7 + timeZone: UTC + maxFileSize: 50MB + +database: + # the name of the JDBC driver, mysql in our case + driverClass: com.mysql.cj.jdbc.Driver + # the username and password + user: openmetadata_user + password: openmetadata_password + # the JDBC URL; the database is called openmetadata_db + url: jdbc:mysql://localhost/openmetadata_db?useSSL=false&serverTimezone=UTC + + +elasticsearch: + host: localhost + port: 9200 + +eventHandlerConfiguration: + eventHandlerClassNames: + - "org.openmetadata.catalog.events.AuditEventHandler" + - "org.openmetadata.catalog.events.ElasticSearchEventHandler" + +health: + delayedShutdownHandlerEnabled: true + shutdownWaitPeriod: 1s + healthCheckUrlPaths: ["/api/v1/health-check"] + healthChecks: + - name: UserDatabaseCheck + critical: true + schedule: + checkInterval: 2500ms + downtimeInterval: 10s + failureAttempts: 2 + successAttempts: 1 +``` + + +## Server Port + +```text +server: + rootPath: '/api/*' + applicationConnectors: + - type: http + port: 8585 + adminConnectors: + - type: http + port: 8586 +``` + +By default OpenMetadata server runs on port 8585. It uses Jetty Server. +The above config can be changed to make it run on a different port. Once you have updated the port details in config restart the server. + + +## Database + +```text +database: + # the name of the JDBC driver, mysql in our case + driverClass: com.mysql.cj.jdbc.Driver + # the username and password + user: openmetadata_user + password: openmetadata_password + # the JDBC URL; the database is called openmetadata_db + url: jdbc:mysql://localhost/openmetadata_db?useSSL=false&serverTimezone=UTC +``` + +The above section is database connection details to MySQL database. +We recommend you to create a MySQL user with strong password and update this section +accordingly. + + +## ElasticSearch + +```text +elasticsearch: + host: localhost + port: 9200 +``` + +ElasticSearch is one of the pre-requisities to run OpenMetadata. +Default configuration expects a single instance of ElasticSearch running on local machine. +Please make sure you update it with your production elastic search. + + +## EventHandlers +```text +eventHandlerConfiguration: + eventHandlerClassNames: + - "org.openmetadata.catalog.events.AuditEventHandler" + - "org.openmetadata.catalog.events.ElasticSearchEventHandler" +``` + +EventHandler configuration is optional. It will update the AuditLog in MySQL DB +and also ElasticSearch indexes whenever any entity is updated either through UI or +API interactions. +We recommend you to leave it there as it enhances the user experience. + + +## Healthcheck + +```text +health: + delayedShutdownHandlerEnabled: true + shutdownWaitPeriod: 1s + healthCheckUrlPaths: ["/api/v1/health-check"] + healthChecks: + - name: UserDatabaseCheck + critical: true + schedule: + checkInterval: 2500ms + downtimeInterval: 10s + failureAttempts: 2 + successAttempts: 1 +``` + + Healthcheck api provides a API endpoint to check the OpenMetadata server health. + We recommend in production settign to use this api to monitor the health of your + OpenMetadata instance. Please tune the above configuration according to your production + needs. + +## Security + +Please follow our [Enable Security Guide](/install/enable-security.md) guide to configure +security for your OpenMetadata installation. + \ No newline at end of file diff --git a/docs/install/run-openmetadata.md b/docs/install/run-openmetadata.md index 2b30f933847..ad6695efd2b 100644 --- a/docs/install/run-openmetadata.md +++ b/docs/install/run-openmetadata.md @@ -25,7 +25,7 @@ docker-compose up ``` -# Run Locally +# Run Manually {% hint style="success" %} This is a quick start guide that will show you how to quickly start a standalone server.