diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md index eb7002ea28d..dddb8f6b21e 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md @@ -558,8 +558,6 @@ site_menu: url: /connectors/ingestion/workflows/dbt/setup-multiple-dbt-projects - category: Connectors / Ingestion / Workflows / dbt / dbt troubleshooting url: /connectors/ingestion/workflows/dbt/dbt-troubleshooting - - category: Connectors / Ingestion / Workflows / Reverse Metadata Ingestion - url: /connectors/ingestion/workflows/reverse-metadata - category: Connectors / Ingestion / Lineage url: /connectors/ingestion/lineage - category: Connectors / Ingestion / Lineage / Edit Data Lineage Manually diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/index.md index 5b945a5c730..b2c4826eb3f 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/athena name="Athena" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "Tags", "dbt", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "Tags", "dbt", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Stored Procedures"] / %} @@ -24,6 +24,7 @@ Configure and schedule Athena metadata and profiler workflows from the OpenMetad - [Troubleshooting](#troubleshooting) - [Workflow Deployment Error](#workflow-deployment-error) - [Related](#related) +- [Reverse Metadata Ingestion](#reverse-metadata-ingestion) {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/athena/yaml"} /%} @@ -240,3 +241,14 @@ Find more information about [Source Identity](https://docs.aws.amazon.com/STS/la {% partial file="/v1.7/connectors/troubleshooting.md" /%} {% partial file="/v1.7/connectors/database/related.md" /%} + +## Reverse Metadata Ingestion + +{% note %} +This feature is specific to Collate and requires the Collate Enterprise License. +{% /note %} + +Athena supports the following reverse metadata ingestion features: +- Table Description updates + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/yaml.md index 3055f461510..b3ea0c9b4e1 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/athena/yaml name="Athena" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "Tags", "dbt", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "Tags", "dbt", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Stored Procedures"] / %} @@ -22,6 +22,7 @@ Configure and schedule Athena metadata and profiler workflows from the OpenMetad - [Data Profiler](#data-profiler) - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) +- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/index.md index 81371132d5c..4e230553525 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/bigquery name="BigQuery" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Stored Procedures", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners"] / %} @@ -24,7 +24,7 @@ Configure and schedule BigQuery metadata and profiler workflows from the OpenMet - [Data Quality](/how-to-guides/data-quality-observability/quality/configure) - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) - +- [Reverse Metadata Ingestion](#reverse-metadata-ingestion) {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/bigquery/yaml"} /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} @@ -163,3 +163,16 @@ We support cross-project lineage, but the data must be ingested within a single {% partial file="/v1.7/connectors/troubleshooting.md" /%} {% partial file="/v1.7/connectors/database/related.md" /%} + +## Reverse Metadata Ingestion + +{% note %} +This feature is specific to Collate and requires the Collate Enterprise License. +{% /note %} + +BigQuery supports the following reverse metadata ingestion features: +- Support for Description updates (Schema, Table) +- Tag management (Schema, Table) + + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/yaml.md index 68ef5d89795..cae61aa8344 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/bigquery/yaml name="BigQuery" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Stored Procedures", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners"] / %} @@ -23,6 +23,7 @@ Configure and schedule BigQuery metadata and profiler workflows from the OpenMet - [Data Profiler](#data-profiler) - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) +- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/index.md index 44ac108e914..7f21ae978ec 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/clickhouse name="Clickhouse" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Tags", "Stored Procedures"] / %} @@ -26,7 +26,7 @@ Configure and schedule Clickhouse metadata and profiler workflows from the OpenM - [Troubleshooting](#troubleshooting) - [Workflow Deployment Error](#workflow-deployment-error) - [Related](#related) - +- [Reverse Metadata Ingestion](#reverse-metadata-ingestion) {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/clickhouse/yaml"} /%} ## Requirements @@ -95,3 +95,14 @@ For the usage and lineage workflow, the user will need `SELECT` privilege. You c {% partial file="/v1.7/connectors/troubleshooting.md" /%} {% partial file="/v1.7/connectors/database/related.md" /%} + +## Reverse Metadata Ingestion + +{% note %} +This feature is specific to Collate and requires the Collate Enterprise License. +{% /note %} + +Clickhouse supports the following reverse metadata ingestion features: +- Support for Description updates (Table, Column) + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/yaml.md index 05fca7cccb1..fd751c1156c 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/clickhouse/yaml name="Clickhouse" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Tags", "Stored Procedures"] / %} @@ -22,6 +22,7 @@ Configure and schedule Clickhouse metadata and profiler workflows from the OpenM - [Data Profiler](#data-profiler) - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) +- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/index.md index 7a71d163dc0..2df013b3a42 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/databricks name="Databricks" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Stored Procedures"] / %} @@ -27,7 +27,7 @@ Configure and schedule Databricks metadata and profiler workflows from the OpenM - [Data Quality](/how-to-guides/data-quality-observability/quality) - [Lineage](/how-to-guides/data-lineage/workflow) - [dbt Integration](/connectors/ingestion/workflows/dbt) - +- [Reverse Metadata Ingestion](#reverse-metadata-ingestion) {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/databricks/yaml"} /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} @@ -75,3 +75,16 @@ If you are using unity catalog in Databricks, then checkout the [Unity Catalog]( {% partial file="/v1.7/connectors/troubleshooting.md" /%} {% partial file="/v1.7/connectors/database/related.md" /%} + +## Reverse Metadata Ingestion + +{% note %} +This feature is specific to Collate and requires the Collate Enterprise License. +{% /note %} + +Databricks supports the following reverse metadata ingestion features: +- Full support for Description updates (Database, Schema, Table, Column) +- Full support for Owner management (Database, Schema, Table) +- Full support for Tag management (Database, Schema, Table, Column) + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/yaml.md index 08774ecc1db..bdef8bf16ef 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/databricks/yaml name="Databricks" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Stored Procedures"] / %} @@ -26,7 +26,7 @@ Configure and schedule Databricks metadata and profiler workflows from the OpenM - [Data Profiler](#data-profiler) - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) - +- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} ## Requirements diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/index.md index d755735cae1..30916155b15 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/mssql name="MSSQL" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Tags", "SSIS packages"] / %} @@ -23,6 +23,7 @@ Configure and schedule MSSQL metadata and profiler workflows from the OpenMetada - [Data Quality](/how-to-guides/data-quality-observability/quality) - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) +- [Reverse Metadata Ingestion](#reverse-metadata-ingestion) {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/mssql/yaml"} /%} @@ -123,3 +124,15 @@ For details step please refer to this [link](https://docs.microsoft.com/en-us/sq {% partial file="/v1.7/connectors/troubleshooting.md" /%} {% partial file="/v1.7/connectors/database/related.md" /%} + +## Reverse Metadata Ingestion + +{% note %} +This feature is specific to Collate and requires the Collate Enterprise License. +{% /note %} + +MSSQL supports the following reverse metadata ingestion features: +- Description updates (Schema, Table, Column) +- Owner management (Database, Schema) + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/yaml.md index a79cdaa1a65..fbef5c6defb 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/mssql/yaml name="MSSQL" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Tags", "SSIS packages"] / %} @@ -22,7 +22,7 @@ Configure and schedule MSSQL metadata and profiler workflows from the OpenMetada - [Data Profiler](#data-profiler) - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) - +- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} ## Requirements diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/index.md index 3952de6538c..63aa2836c58 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/mysql name="MySQL" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Data Profiler", "Data Quality", "dbt", "View Lineage", "View Column-level Lineage", "Query Usage", "Sample Data"] +availableFeatures=["Metadata", "Data Profiler", "Data Quality", "dbt", "View Lineage", "View Column-level Lineage", "Query Usage", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Tags", "Stored Procedures"] / %} @@ -22,7 +22,7 @@ Configure and schedule MySQL metadata and profiler workflows from the OpenMetada - [dbt Integration](/connectors/ingestion/workflows/dbt) - [Enable Security](#securing-mysql-connection-with-ssl-in-openmetadata) - [Data Lineage](/how-to-guides/data-lineage/workflow) - +- [Reverse Metadata Ingestion](#reverse-metadata-ingestion) {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/mysql/yaml"} /%} ## Requirements @@ -149,7 +149,6 @@ Executing the profiler workflow or data quality tests, will require the user to {%note%} When using Assume Role authentication, ensure you provide the following details: - **AWS Region**: Specify the AWS region for your deployment. - - **Assume Role ARN**: Provide the ARN of the role in your AWS account that OpenMetadata will assume. {%/note%} - **Assume Role Session Name**: An identifier for the assumed role session. Use the role session name to uniquely identify a session when the same role @@ -195,3 +194,24 @@ To establish secure connections between OpenMetadata and MySQL, navigate to the {% partial file="/v1.7/connectors/troubleshooting.md" /%} {% partial file="/v1.7/connectors/database/related.md" /%} + +## Reverse Metadata Ingestion + +{% note %} +This feature is specific to Collate and requires the Collate Enterprise License. +{% /note %} + +MySQL supports the following reverse metadata ingestion features: +- Support for table description updates + +### Requirements for Reverse Metadata + +In addition to the basic ingestion requirements, for reverse metadata ingestion the user needs: +- `ALTER` privileges on tables to update descriptions + +```sql +-- Grant required privileges for reverse metadata +GRANT ALTER ON DATABASE_NAME.TABLE_NAME TO 'USER_NAME'@'%'; +``` + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md index fe3ba2b0e82..1c5a9ba0d0c 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/mysql/yaml name="MySQL" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Data Profiler", "Data Quality", "dbt", "View Lineage", "View Column-level Lineage", "Query Usage", "Sample Data"] +availableFeatures=["Metadata", "Data Profiler", "Data Quality", "dbt", "View Lineage", "View Column-level Lineage", "Query Usage", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Tags", "Stored Procedures"] / %} @@ -22,7 +22,7 @@ Configure and schedule MySQL metadata and profiler workflows from the OpenMetada - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) - [Enable Security](#securing-mysql-connection-with-ssl-in-openmetadata) - +- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} ## Requirements diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/index.md index 90a19774ea1..3ead3e0f685 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/oracle name="Oracle" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Tags"] / %} @@ -21,7 +21,7 @@ Configure and schedule Oracle metadata and profiler workflows from the OpenMetad - [Data Quality](/how-to-guides/data-quality-observability/quality) - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) - +- [Reverse Metadata Ingestion](#reverse-metadata-ingestion) {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/oracle/yaml"} /%} ## Requirements @@ -133,3 +133,15 @@ there is no routine out of the box in Oracle to grant SELECT to a full schema. {% partial file="/v1.7/connectors/troubleshooting.md" /%} {% partial file="/v1.7/connectors/database/related.md" /%} + +## Reverse Metadata Ingestion + +{% note %} +This feature is specific to Collate and requires the Collate Enterprise License. +{% /note %} + +Oracle supports the following reverse metadata ingestion features: +- Support for Description updates (Table, Column) + + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/yaml.md index dfc943e92ef..8e89b7c127f 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/oracle/yaml name="Oracle" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Tags"] / %} @@ -21,7 +21,7 @@ Configure and schedule Oracle metadata and profiler workflows from the OpenMetad - [Data Quality](#data-quality) - [Lineage](#lineage) - [dbt Integration](#dbt-integration) - +- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} ## Requirements diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/index.md index 9fde79bc909..520f4eca0b9 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/postgres name="PostgreSQL" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Owners", "Tags", "Stored Procedures", "Sample Data", "Stored Procedures Lineage"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Owners", "Tags", "Stored Procedures", "Sample Data", "Stored Procedures Lineage", "Reverse Metadata Ingestion"] unavailableFeatures=[] / %} @@ -23,6 +23,7 @@ Configure and schedule PostgreSQL metadata and profiler workflows from the OpenM - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) - [Enable Security](#securing-postgres-connection-with-ssl-in-openmetadata) +- [Reverse Metadata Ingestion](#reverse-metadata-ingestion) {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/postgres/yaml"} /%} @@ -217,3 +218,25 @@ For IAM authentication, it is recommended to choose the `allow` mode or another {% partial file="/v1.7/connectors/troubleshooting.md" /%} {% partial file="/v1.7/connectors/database/related.md" /%} + +## Reverse Metadata Ingestion + +{% note %} +This feature is specific to Collate and requires the Collate Enterprise License. +{% /note %} + +PostgreSQL supports the following reverse metadata ingestion features: +- Support for Description updates (Database, Schema, Table, Column) +- Owner management (Database, Schema, Table) + +### Requirements for Reverse Metadata + +In addition to the basic ingestion requirements, for reverse metadata ingestion the user needs: +- `SUPERUSER` privilege to the user + +```sql +-- Grant superuser privilege to the user +ALTER ROLE USER_NAME SUPERUSER; +``` + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/yaml.md index 0a902d9bfe3..b76017cb249 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/postgres/yaml name="PostgreSQL" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Owners", "Tags", "Sample Data", "Stored Procedures"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Owners", "Tags", "Sample Data", "Stored Procedures", "Reverse Metadata Ingestion"] unavailableFeatures=[] / %} @@ -23,7 +23,7 @@ Configure and schedule PostgreSQL metadata and profiler workflows from the OpenM - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) - [Enable Security](#securing-postgres-connection-with-ssl-in-openmetadata) - +- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} ## Requirements diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/index.md index fa73b63e7f1..a6a25a2e15d 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/redshift name="Redshift" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Tags"] / %} @@ -24,7 +24,7 @@ Configure and schedule Redshift metadata and profiler workflows from the OpenMet - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) - [Enable Security](#securing-redshift-connection-with-ssl-in-openmetadata) - +- [Reverse Metadata Ingestion](#reverse-metadata-ingestion) {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/redshift/yaml"} /%} ## Requirements @@ -130,3 +130,15 @@ Under `Advanced Config`, specify the SSL mode appropriate for your connection, s {% partial file="/v1.7/connectors/troubleshooting.md" /%} {% partial file="/v1.7/connectors/database/related.md" /%} + +## Reverse Metadata Ingestion + +{% note %} +This feature is specific to Collate and requires the Collate Enterprise License. +{% /note %} + +Redshift supports the following reverse metadata ingestion features: +- Full support for Description updates (Database, Schema, Table, Column) +- Owner management (Database, Schema, Table) + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/yaml.md index d71f2a34bb4..74e54a06a9a 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/redshift/yaml name="Redshift" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Tags"] / %} @@ -24,6 +24,7 @@ Configure and schedule Redshift metadata and profiler workflows from the OpenMet - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) - [Enable Security](#securing-redshift-connection-with-ssl-in-openmetadata) +- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/index.md index 69540e0f471..445d3636f6d 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/snowflake name="Snowflake" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Stored Procedures", "Tags", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Stored Procedures", "Tags", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners"] / %} @@ -24,6 +24,7 @@ Configure and schedule Snowflake metadata and profiler workflows from the OpenMe - [Data Quality](/how-to-guides/data-quality-observability/quality) - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) +- [Reverse Metadata Ingestion](#reverse-metadata-ingestion) {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/snowflake/yaml"} /%} @@ -85,7 +86,6 @@ You can find more information about the `account_usage` schema [here](https://do - **Ingesting Stored Procedures**: Openmetadata fetches the information by querying `snowflake.account_usage.procedures` & `snowflake.account_usage.functions`. - ## Metadata Ingestion {% partial @@ -151,3 +151,59 @@ GRANT APPLY TAG TO ROLE NEW_ROLE; Depending on your view ddl you can grant the relevant privileged as per above queries. {% partial file="/v1.7/connectors/database/related.md" /%} + + +## Reverse Metadata Ingestion + +{% note %} +This feature is specific to Collate and requires the Collate Enterprise License. +{% /note %} + +Snowflake supports the following reverse metadata ingestion features: +- Full support for Description updates (Database, Schema, Table, Column) +- Tag management (Schema, Table, Column) +- Automated masking policy application based on tags +- Integration with auto-classification workflows + +### Requirements for Reverse Metadata Ingestion + +In addition to the basic ingestion requirements, for reverse metadata ingestion the user needs: +- `ACCOUNTADMIN` role or a role with similar privileges to modify descriptions and tags +- Access to `snowflake.account_usage.tag_references` for tag management + +```sql +-- Grant modify database privileges to the role +GRANT MODIFY ON DATABASE DATABASE_NAME TO ROLE NEW_ROLE; + +-- Grant ownership on schema to the role +GRANT OWNERSHIP ON SCHEMA DATABASE_NAME.SCHEMA_NAME TO ROLE NEW_ROLE REVOKE CURRENT GRANTS; + +-- Grant ownership on table to the role +GRANT OWNERSHIP ON TABLE DATABASE_NAME.SCHEMA_NAME.TABLE_NAME TO ROLE NEW_ROLE REVOKE CURRENT GRANTS; + +-- Grant imported privileges on snowflake database to the role +GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE NEW_ROLE; +``` + +### Automated Masking Policies with Tags + +You can configure masking policies in Snowflake to be automatically applied when specific tags are set through OpenMetadata. For example, when you apply a `Sensitive` tag to a column in OpenMetadata, the corresponding masking policy will be automatically applied to that column in Snowflake. + +```sql +-- Create masking policy +CREATE MASKING POLICY SENSITIVE_DATA AS (VAL STRING) RETURNS STRING -> CASE WHEN VAL IS NOT NULL THEN '**********' ELSE NULL END; + +-- Apply masking policy to PII.Sensitive tag +ALTER TAG PII SET MASKING POLICY SENSITIVE_DATA; +``` + +### Auto-Classification Integration + +The reverse ingestion workflow can be combined with OpenMetadata's auto-classification feature. When you run the auto-classification workflow: +1. Tags will be automatically identified and added in OpenMetadata +2. These tags will then be synchronized back to Snowflake through the reverse ingestion process + +This creates a seamless workflow for identifying and protecting sensitive data across both platforms. + + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Ingestion Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/yaml.md index 2090f1bb93e..020a421a8a2 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/snowflake/yaml name="Snowflake" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Stored Procedures", "Tags", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Stored Procedures", "Tags", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners"] / %} @@ -23,6 +23,7 @@ Configure and schedule Snowflake metadata and profiler workflows from the OpenMe - [Data Profiler](#data-profiler) - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) +- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/index.md index c5e41ba9e00..1371887e2fa 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/unity-catalog name="Unity Catalog" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Tags", "Stored Procedures"] / %} @@ -21,7 +21,7 @@ Configure and schedule Unity Catalog metadata workflow from the OpenMetadata UI: - [Data Quality](/how-to-guides/data-quality-observability/quality) - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) - +- [Reverse Metadata Ingestion](#reverse-metadata-ingestion) {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/unity-catalog/yaml"} /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} @@ -65,3 +65,16 @@ Configure and schedule Unity Catalog metadata workflow from the OpenMetadata UI: {% partial file="/v1.7/connectors/troubleshooting.md" /%} {% partial file="/v1.7/connectors/database/related.md" /%} + +## Reverse Metadata Ingestion + +{% note %} +This feature is specific to Collate and requires the Collate Enterprise License. +{% /note %} + +Unity Catalog supports the following reverse metadata ingestion features: +- Full support for Description updates (Database, Schema, Table, Column) +- Full support for Owner management (Database, Schema, Table) +- Full support for Tag management (Database, Schema, Table, Column) + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/yaml.md index ce0ef442dea..98d8300fa43 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/unity-catalog/yaml name="Unity Catalog" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Sample Data"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Sample Data", "Reverse Metadata Ingestion"] unavailableFeatures=["Owners", "Tags", "Stored Procedures"] / %} @@ -20,7 +20,7 @@ Configure and schedule Unity Catalog metadata workflow from the OpenMetadata UI: - [Query Usage](#query-usage) - [Lineage](#lineage) - [dbt Integration](#dbt-integration) - +- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} ## Requirements diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/ingestion/workflows/reverse-metadata.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/ingestion/workflows/reverse-metadata.md index e6058f3f7f0..d749eaaca19 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/ingestion/workflows/reverse-metadata.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/ingestion/workflows/reverse-metadata.md @@ -1,10 +1,12 @@ --- title: Reverse Metadata Ingestion Collate | Feature & Configuration slug: /connectors/ingestion/workflows/reverse-metadata -collate: true --- # Reverse Metadata Ingestion +{% note %} +This feature is specific to Collate and requires the Collate Enterprise License. +{% /note %} Reverse metadata ingestion is an advanced feature in OpenMetadata that facilitates bi-directional synchronization between OpenMetadata and the source database systems. While standard ingestion pulls metadata into OpenMetadata, reverse ingestion enables pushing metadata changes made within OpenMetadata back to the source systems. This ensures consistency and alignment across the entire data infrastructure. diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/menu.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/menu.md index ac0250a9715..7c80f0d5fd1 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/menu.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/menu.md @@ -702,6 +702,9 @@ site_menu: - category: Connectors / Ingestion / Workflows/ Metadata / Multithreading url: /connectors/ingestion/workflows/metadata/multithreading + - category: Connectors / Ingestion / Workflows / Reverse Metadata Ingestion + url: /connectors/ingestion/workflows/reverse-metadata + - category: Connectors / Ingestion / Workflows / Usage url: /connectors/ingestion/workflows/usage - category: Connectors / Ingestion / Workflows / Usage / Usage Workflow Through Query Logs