From c4a89a2cabb6bb6366ad517347007f70255e63a6 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 12 Aug 2021 13:53:29 -0700 Subject: [PATCH] Update ingestion connector docs --- .../metadata-ingestion/connectors/mssql.md | 73 +++++++++++++++++-- .../metadata-ingestion/connectors/mysql.md | 63 +++++++++++++++- .../metadata-ingestion/connectors/postgres.md | 24 ++++-- .../metadata-ingestion/ingest-sample-data.md | 20 ++--- 4 files changed, 154 insertions(+), 26 deletions(-) diff --git a/docs/install/metadata-ingestion/connectors/mssql.md b/docs/install/metadata-ingestion/connectors/mssql.md index 28399f0c12b..d2d35731696 100644 --- a/docs/install/metadata-ingestion/connectors/mssql.md +++ b/docs/install/metadata-ingestion/connectors/mssql.md @@ -12,21 +12,28 @@ description: This guide will help install MsSQL connector and run manually OpenMetadata is built using Java, DropWizard, Jetty, and MySQL. 1. Python 3.7 or above -2. Create and activate python env - - ```bash - python3 -m venv env - source env/bin/activate - ``` {% endhint %} -\*\*\*\* -## Install +### Install from PyPI or Source +{% tabs %} +{% tab title="Install Using PyPI" %} ```bash +pip install 'openmetadata-ingestion[mssql]' +``` +{% endtab %} +{% tab title="Build from source " %} +```bash +# checkout OpenMetadata +git clone https://github.com/open-metadata/OpenMetadata.git +cd OpenMetadata/ingestion +python3 -m venv env +source env/bin/activate pip install '.[mssql]' ``` +{% endtab %} +{% endtabs %} ## Run Manually @@ -64,3 +71,53 @@ metadata ingest -c ./pipelines/mssql.json 5. **table\_pattern** - It contains allow, deny options to choose which pattern of datasets you want to ingest into OpenMetadata. 6. **database** - \_\*\*\_Database name from where data is to be fetched from. +## Publish to OpenMetadata + +{% code title="mssql.json" %} +```javascript +{ + "source": { + "type": "mssql", + "config": { + "host_port": "localhost:1433", + "service_name": "local_mssql", + "service_type": "MSSQL", + "database":"catalog_test", + "username": "sa", + "password": "test!Password", + "include_pattern": { + "excludes": ["catalog_test.*"] + } + } + }, + "processor": { + "type": "pii-tags", + "config": { + } + }, + "sink": { + "type": "metadata-rest-tables", + "config": { + } + }, + "metadata_server": { + "type": "metadata-server", + "config": { + "api_endpoint": "http://localhost:8585/api", + "auth_provider_type": "no-auth" + } + }, + "cron": { + "minute": "*/5", + "hour": null, + "day": null, + "month": null, + "day_of_week": null + } +} + ... +``` +{% endcode %} + +Add Optional ```pii-tags``` processor +and ```metadata-rest-tables``` sink along with ```metadata-server``` config \ No newline at end of file diff --git a/docs/install/metadata-ingestion/connectors/mysql.md b/docs/install/metadata-ingestion/connectors/mysql.md index 7be6f769890..f60fdbdc666 100644 --- a/docs/install/metadata-ingestion/connectors/mysql.md +++ b/docs/install/metadata-ingestion/connectors/mysql.md @@ -20,11 +20,25 @@ OpenMetadata is built using Java, DropWizard, Jetty, and MySQL. ``` {% endhint %} -### Install +### Install from PyPI or Source +{% tabs %} +{% tab title="Install Using PyPI" %} ```bash +pip install 'openmetadata-ingestion[mysql]' +``` +{% endtab %} +{% tab title="Build from source " %} +```bash +# checkout OpenMetadata +git clone https://github.com/open-metadata/OpenMetadata.git +cd OpenMetadata/ingestion +python3 -m venv env +source env/bin/activate pip install '.[mysql]' ``` +{% endtab %} +{% endtabs %} ### Run Manually @@ -57,3 +71,50 @@ metadata ingest -c ./pipelines/mysql.json 3. **service\_name** - Service Name for this MySQL cluster. If you added MySQL cluster through OpenMetadata UI, make sure the service name matches the same. 4. **table\_pattern** - It contains allow, deny options to choose which pattern of datasets you want to ingest into OpenMetadata +## Publish to OpenMetadata + +{% code title="mysql.json" %} +```javascript +{ + "source": { + "type": "mysql", + "config": { + "username": "openmetadata_user", + "password": "openmetadata_password", + "service_name": "local_mysql", + "service_type": "MySQL", + "include_pattern": { + "excludes": ["mysql.*", "information_schema.*"] + } + } + }, + "processor": { + "type": "pii-tags", + "config": { + "api_endpoint": "http://localhost:8585/api" + } + }, + "sink": { + "type": "metadata-rest-tables", + "config": { + } + }, + "metadata_server": { + "type": "metadata-server", + "config": { + "api_endpoint": "http://localhost:8585/api", + "auth_provider_type": "no-auth" + } + }, + "cron": { + "minute": "*/5", + "hour": null, + "day": null, + "month": null, + "day_of_week": null + } +} +``` +{% endcode %} + + diff --git a/docs/install/metadata-ingestion/connectors/postgres.md b/docs/install/metadata-ingestion/connectors/postgres.md index 7356f2600f2..6ab23e89fec 100644 --- a/docs/install/metadata-ingestion/connectors/postgres.md +++ b/docs/install/metadata-ingestion/connectors/postgres.md @@ -12,19 +12,29 @@ description: This guide will help install Postgres connector and run manually OpenMetadata is built using Java, DropWizard, Jetty, and MySQL. 1. Python 3.7 or above -2. Create and activate python env - - ```bash - python3 -m venv env - source env/bin/activate - ``` {% endhint %} -### Install +### Install from PyPI or Source +{% tabs %} +{% tab title="Install Using PyPI" %} ```bash +pip install 'openmetadata-ingestion[postgres]' +``` +{% endtab %} +{% tab title="Build from source " %} +```bash +# checkout OpenMetadata +git clone https://github.com/open-metadata/OpenMetadata.git +cd OpenMetadata/ingestion +python3 -m venv env +source env/bin/activate pip install '.[postgres]' ``` +{% endtab %} +{% endtabs %} + + ### Run Manually diff --git a/docs/install/metadata-ingestion/ingest-sample-data.md b/docs/install/metadata-ingestion/ingest-sample-data.md index 9500f0cfa78..69aca2f476b 100644 --- a/docs/install/metadata-ingestion/ingest-sample-data.md +++ b/docs/install/metadata-ingestion/ingest-sample-data.md @@ -15,31 +15,32 @@ OpenMetadata is built using Java, DropWizard, Jetty, and MySQL. 2. Create and activate python env ```bash - python3 -m venv env - source env/bin/activate - ``` + ``` {% endhint %} -### Build from source or PyPI +### Install from PyPI or Source {% tabs %} +{% tab title="Install Using PyPI" %} +```bash +pip install 'openmetadata-ingestion[sample-tables, elasticsearch]' +``` +{% endtab %} {% tab title="Build from source " %} ```bash # checkout OpenMetadata git clone https://github.com/open-metadata/OpenMetadata.git cd OpenMetadata/ingestion +python3 -m venv env +source env/bin/activate +pip install '.[sample-tables, elasticsearch]' ``` -{% endtab %} - -{% tab title="Install Using PyPI" %} - {% endtab %} {% endtabs %} ### Ingest sample tables and users ```bash -pip install '.[sample-tables]' metadata ingest -c ./pipelines/sample_tables.json metadata ingest -c ./pipelines/sample_users.json ``` @@ -55,7 +56,6 @@ docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elas Index sample data in ElasticSearch: ```bash -pip install '.[elasticsearch]' metadata ingest -c ./pipelines/metadata_to_es.json ```