From f633514bbeeab9c3293e9a71a0079b1df88a66f0 Mon Sep 17 00:00:00 2001 From: Keshav Mohta <68001229+keshavmohta09@users.noreply.github.com> Date: Thu, 10 Apr 2025 11:03:42 +0530 Subject: [PATCH] Fixes: Reverse Metadata Documentation (#20726) * fix: make reverse metadata docs visible to collate * fix: reverse metadata ingestion to reverse metadata --- .../database/athena/reverse-metadata.md | 29 +++++++ .../database/bigquery/reverse-metadata.md | 34 ++++++++ .../database/clickhouse/reverse-metadata.md | 30 +++++++ .../database/databricks/reverse-metadata.md | 43 ++++++++++ .../database/mssql/reverse-metadata.md | 53 ++++++++++++ .../database/mysql/reverse-metadata.md | 34 ++++++++ .../database/oracle/reverse-metadata.md | 31 +++++++ .../database/postgres/reverse-metadata.md | 45 ++++++++++ .../database/redshift/reverse-metadata.md | 36 ++++++++ .../database/snowflake/reverse-metadata.md | 72 ++++++++++++++++ .../database/unitycatalog/reverse-metadata.md | 43 ++++++++++ .../v1.7/connectors/reverse-metadata-link.md | 1 + .../reverse-metadata-workflow-link.md | 1 + .../content/v1.7.x-SNAPSHOT/collate-menu.md | 2 + .../connectors/database/athena/index.md | 42 ++-------- .../connectors/database/athena/yaml.md | 4 +- .../connectors/database/bigquery/index.md | 45 +--------- .../connectors/database/bigquery/yaml.md | 4 +- .../connectors/database/clickhouse/index.md | 41 +-------- .../connectors/database/clickhouse/yaml.md | 4 +- .../connectors/database/databricks/index.md | 50 +---------- .../connectors/database/databricks/yaml.md | 4 +- .../connectors/database/mssql/index.md | 64 +------------- .../connectors/database/mssql/yaml.md | 4 +- .../connectors/database/mysql/index.md | 45 +--------- .../connectors/database/mysql/yaml.md | 4 +- .../connectors/database/oracle/index.md | 41 +-------- .../connectors/database/oracle/yaml.md | 4 +- .../connectors/database/postgres/index.md | 56 +------------ .../connectors/database/postgres/yaml.md | 4 +- .../connectors/database/redshift/index.md | 48 ++--------- .../connectors/database/redshift/yaml.md | 4 +- .../connectors/database/snowflake/index.md | 83 +------------------ .../connectors/database/snowflake/yaml.md | 4 +- .../database/unity-catalog/index.md | 43 +--------- .../connectors/database/unity-catalog/yaml.md | 5 +- .../ingestion/workflows/reverse-metadata.md | 27 +++--- .../content/v1.7.x-SNAPSHOT/menu.md | 3 - 38 files changed, 537 insertions(+), 550 deletions(-) create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/database/athena/reverse-metadata.md create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/database/bigquery/reverse-metadata.md create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/database/clickhouse/reverse-metadata.md create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/database/databricks/reverse-metadata.md create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/database/mssql/reverse-metadata.md create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/database/mysql/reverse-metadata.md create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/database/oracle/reverse-metadata.md create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/database/postgres/reverse-metadata.md create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/database/redshift/reverse-metadata.md create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/database/snowflake/reverse-metadata.md create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/database/unitycatalog/reverse-metadata.md create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/reverse-metadata-link.md create mode 100644 openmetadata-docs/content/partials/v1.7/connectors/reverse-metadata-workflow-link.md diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/athena/reverse-metadata.md b/openmetadata-docs/content/partials/v1.7/connectors/database/athena/reverse-metadata.md new file mode 100644 index 00000000000..32bf50f0333 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/database/athena/reverse-metadata.md @@ -0,0 +1,29 @@ +## Reverse Metadata + +### Description Management + +Athena supports description updates at the following level: +- Table level + +### Owner Management + +❌ Owner management is not supported for Athena. + +### Tag Management + +❌ Tag management is not supported for Athena. + +### Custom SQL Template + +Athena supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. + +Here are examples of custom SQL queries for metadata changes: + +```sql +-- Update table description +ALTER TABLE {schema}.{table} SET TBLPROPERTIES ('comment' = {description}); +``` + +The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/bigquery/reverse-metadata.md b/openmetadata-docs/content/partials/v1.7/connectors/database/bigquery/reverse-metadata.md new file mode 100644 index 00000000000..a4a3cbeaff7 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/database/bigquery/reverse-metadata.md @@ -0,0 +1,34 @@ +## Reverse Metadata + + + +### Description Management + +BigQuery supports description updates at the following levels: +- Schema level +- Table level + +### Owner Management + +❌ Owner management is not supported for BigQuery. + +### Tag Management + +BigQuery supports tag management at the following levels: +- Schema level +- Table level + +### Custom SQL Template + +BigQuery supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. + +Here are examples of custom SQL queries for metadata changes: + +```sql +-- Update table labels +ALTER TABLE `{database}.{schema}.{table}` SET OPTIONS (labels = {tags}); +``` + +The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/clickhouse/reverse-metadata.md b/openmetadata-docs/content/partials/v1.7/connectors/database/clickhouse/reverse-metadata.md new file mode 100644 index 00000000000..420191a22b2 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/database/clickhouse/reverse-metadata.md @@ -0,0 +1,30 @@ +## Reverse Metadata + +### Description Management + +Clickhouse supports description updates at the following levels: +- Table level +- Column level + +### Owner Management + +❌ Owner management is not supported for Clickhouse. + +### Tag Management + +❌ Tag management is not supported for Clickhouse. + +### Custom SQL Template + +Clickhouse supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. + +Here are examples of custom SQL queries for metadata changes: + +```sql +-- Update column description +ALTER TABLE `{schema}`.{table} MODIFY COLUMN {column} COMMENT {description}; +``` + +The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/databricks/reverse-metadata.md b/openmetadata-docs/content/partials/v1.7/connectors/database/databricks/reverse-metadata.md new file mode 100644 index 00000000000..e5045256bca --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/database/databricks/reverse-metadata.md @@ -0,0 +1,43 @@ +## Reverse Metadata + +### Description Management + +Databricks supports description updates at all levels: +- Database level +- Schema level +- Table level +- Column level + +### Owner Management + +Databricks supports owner management at the following levels: +- Database level +- Schema level +- Table level + +{% note %} +Databricks does not support to set `null` as owner. +{% /note %} + +### Tag Management + +Databricks supports tag management at all levels: +- Database level +- Schema level +- Table level +- Column level + +### Custom SQL Template + +Databricks supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. + +Here are examples of custom SQL queries for metadata changes: + +```sql +-- Set table tags +ALTER TABLE {database}.{schema}.{table} SET TAGS {tags}; +``` + +The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/mssql/reverse-metadata.md b/openmetadata-docs/content/partials/v1.7/connectors/database/mssql/reverse-metadata.md new file mode 100644 index 00000000000..17488d560a5 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/database/mssql/reverse-metadata.md @@ -0,0 +1,53 @@ +## Reverse Metadata + +### Description Management + +MSSQL supports description updates at the following levels: +- Schema level +- Table level +- Column level + +### Owner Management + +MSSQL supports owner management at the following levels: +- Database level +- Schema level + +### Tag Management + +❌ Tag management is not supported for MSSQL. + +### Custom SQL Template + +MSSQL supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. + +Here are examples of custom SQL queries for metadata changes: + +```sql +-- Update schema description +IF NOT EXISTS (SELECT 1 + FROM {database}.sys.extended_properties + WHERE name = N'MS_Description' And + major_id = SCHEMA_ID('{schema}') AND + minor_id = 0 + ) +BEGIN + EXEC {database}.{schema}.sp_addextendedproperty + @name = N'MS_Description', + @value = N{description}, + @level0type = N'SCHEMA', + @level0name = N'{schema}' +END +ELSE +BEGIN + EXEC {database}.{schema}.sp_updateextendedproperty + @name = N'MS_Description', + @value = N{description}, + @level0type = N'SCHEMA', + @level0name = N'{schema}' +END; +``` + +The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/mysql/reverse-metadata.md b/openmetadata-docs/content/partials/v1.7/connectors/database/mysql/reverse-metadata.md new file mode 100644 index 00000000000..0bad1467824 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/database/mysql/reverse-metadata.md @@ -0,0 +1,34 @@ +## Reverse Metadata + +### Description Management + +MySQL supports description updates at the following level: +- Table level + +### Owner Management + +Owner management is not supported for MySQL. + +### Tag Management + +❌ Tag management is not supported for MySQL. + +### Custom SQL Template + +MySQL supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. + +Here are examples of custom SQL queries for metadata changes: + +```sql +-- Update table description +ALTER TABLE {schema}.{table} COMMENT = {description}; +``` + +The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). + +### Requirements for Reverse Metadata + +In addition to the basic ingestion requirements, for reverse metadata ingestion the user needs: +- ` diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/oracle/reverse-metadata.md b/openmetadata-docs/content/partials/v1.7/connectors/database/oracle/reverse-metadata.md new file mode 100644 index 00000000000..46a7c9e983d --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/database/oracle/reverse-metadata.md @@ -0,0 +1,31 @@ +## Reverse Metadata + + +### Description Management + +Oracle supports description updates at the following levels: +- Table level +- Column level + +### Owner Management + +❌ Owner management is not supported for Oracle. + +### Tag Management + +❌ Tag management is not supported for Oracle. + +### Custom SQL Template + +Oracle supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. + +Here are examples of custom SQL queries for metadata changes: + +```sql +-- Update column description +COMMENT ON COLUMN {schema}.{table}.{column} IS {description} +``` + +The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/postgres/reverse-metadata.md b/openmetadata-docs/content/partials/v1.7/connectors/database/postgres/reverse-metadata.md new file mode 100644 index 00000000000..4da7378f3fe --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/database/postgres/reverse-metadata.md @@ -0,0 +1,45 @@ +## Reverse Metadata + +### Description Management + +PostgreSQL supports description updates at all levels: +- Database level +- Schema level +- Table level +- Column level + +### Owner Management + +PostgreSQL supports owner management at the following levels: +- Database level +- Schema level +- Table level + +### Tag Management + +❌ Tag management is not supported for PostgreSQL. + +### Custom SQL Template + +PostgreSQL supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. + +Here are examples of custom SQL queries for metadata changes: + +```sql +-- Update database owner +ALTER DATABASE {database} OWNER TO {owner}; +``` + +The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). + +### Requirements for Reverse Metadata + +In addition to the basic ingestion requirements, for reverse metadata ingestion the user needs: +- `SUPERUSER` privilege to the user + +```sql +-- Grant superuser privilege to the user +ALTER ROLE USER_NAME SUPERUSER; +``` diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/redshift/reverse-metadata.md b/openmetadata-docs/content/partials/v1.7/connectors/database/redshift/reverse-metadata.md new file mode 100644 index 00000000000..9f43dd02328 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/database/redshift/reverse-metadata.md @@ -0,0 +1,36 @@ +## Reverse Metadata + + +### Description Management + +Redshift supports description updates at all levels: +- Database level +- Schema level +- Table level +- Column level + +### Owner Management + +Redshift supports owner management at the following levels: +- Database level +- Schema level +- Table level + +### Tag Management + +❌ Tag management is not supported for Redshift. + +### Custom SQL Template + +Redshift supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. + +Here are examples of custom SQL queries for metadata changes: + +```sql +-- Update column description +COMMENT ON COLUMN {database}.{schema}.{table}.{column} IS {description}; +``` + +The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/snowflake/reverse-metadata.md b/openmetadata-docs/content/partials/v1.7/connectors/database/snowflake/reverse-metadata.md new file mode 100644 index 00000000000..ae672398277 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/database/snowflake/reverse-metadata.md @@ -0,0 +1,72 @@ + +## Reverse Metadata + +### Description Management + +Snowflake supports full description updates at all levels: +- Database level +- Schema level +- Table level +- Column level + +### Owner Management + +❌ Owner management is not supported for Snowflake. + +### Tag Management + +Snowflake supports tag management at the following levels: +- Schema level +- Table level +- Column level + +Additionally, Snowflake supports automated masking policy application based on tags. For example, when you apply a `Sensitive` tag to a column in OpenMetadata, the corresponding masking policy will be automatically applied to that column in Snowflake. + +{% note %} +While OpenMetadata differentiates between "classification" values and "tag" values, Snowflake's masking policies only support actions based on classification values (tag keys). Therefore, it is advised not to use PII.None along with tag masking policies. +{% /note %} + +```sql +-- Create masking policy +CREATE MASKING POLICY SENSITIVE_DATA AS (VAL STRING) RETURNS STRING -> CASE WHEN VAL IS NOT NULL THEN '**********' ELSE NULL END; + +-- Apply masking policy to PII.Sensitive tag +ALTER TAG PII SET MASKING POLICY SENSITIVE_DATA; +``` + +### Custom SQL Template + +Snowflake supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. + +Here are examples of custom SQL queries for metadata changes: + +```sql +-- Set schema tag +ALTER SCHEMA {database}.{schema} SET TAG {database}.{schema}.{tag_key} = '{tag_value}'; +``` + +The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). + +### Requirements for Reverse Metadata + +In addition to the basic ingestion requirements, for reverse metadata ingestion the user needs: +- `ACCOUNTADMIN` role or a role with similar privileges to modify descriptions and tags +- Access to `snowflake.account_usage.tag_references` for tag management + +```sql +-- Grant modify database privileges to the role +GRANT MODIFY ON DATABASE DATABASE_NAME TO ROLE NEW_ROLE; + +-- Grant ownership on schema to the role +GRANT OWNERSHIP ON SCHEMA DATABASE_NAME.SCHEMA_NAME TO ROLE NEW_ROLE REVOKE CURRENT GRANTS; + +-- Grant ownership on table to the role +GRANT OWNERSHIP ON TABLE DATABASE_NAME.SCHEMA_NAME.TABLE_NAME TO ROLE NEW_ROLE REVOKE CURRENT GRANTS; + +-- Grant imported privileges on snowflake database to the role +GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE NEW_ROLE; +``` + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/partials/v1.7/connectors/database/unitycatalog/reverse-metadata.md b/openmetadata-docs/content/partials/v1.7/connectors/database/unitycatalog/reverse-metadata.md new file mode 100644 index 00000000000..daa43834488 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/database/unitycatalog/reverse-metadata.md @@ -0,0 +1,43 @@ +## Reverse Metadata + +### Description Management + +Unity Catalog supports description updates at all levels: +- Database level +- Schema level +- Table level +- Column level + +### Owner Management + +Unity Catalog supports owner management at the following levels: +- Database level +- Schema level +- Table level + +{% note %} +Unity Catalog does not support to set `null` as owner. +{% /note %} + +### Tag Management + +Unity Catalog supports tag management at all levels: +- Database level +- Schema level +- Table level +- Column level + +### Custom SQL Template + +Unity Catalog supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. + +Here are examples of custom SQL queries for metadata changes: + +```sql +-- Set table tags +ALTER TABLE {database}.{schema}.{table} SET TAGS {tags}; +``` + +The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). + +For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/partials/v1.7/connectors/reverse-metadata-link.md b/openmetadata-docs/content/partials/v1.7/connectors/reverse-metadata-link.md new file mode 100644 index 00000000000..b0f8be1ea04 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/reverse-metadata-link.md @@ -0,0 +1 @@ +- [Reverse Metadata](#reverse-metadata) \ No newline at end of file diff --git a/openmetadata-docs/content/partials/v1.7/connectors/reverse-metadata-workflow-link.md b/openmetadata-docs/content/partials/v1.7/connectors/reverse-metadata-workflow-link.md new file mode 100644 index 00000000000..6a0fbd7db46 --- /dev/null +++ b/openmetadata-docs/content/partials/v1.7/connectors/reverse-metadata-workflow-link.md @@ -0,0 +1 @@ +- [Reverse Metadata](/connectors/ingestion/workflows/reverse-metadata) \ No newline at end of file diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md index 4e6553e8592..866835ba547 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/collate-menu.md @@ -682,6 +682,8 @@ site_menu: url: /connectors/ingestion/workflows/dbt/setup-multiple-dbt-projects - category: Connectors / Ingestion / Workflows / dbt / dbt troubleshooting url: /connectors/ingestion/workflows/dbt/dbt-troubleshooting + - category: Connectors / Ingestion / Workflows / Reverse Metadata Ingestion + url: /connectors/ingestion/workflows/reverse-metadata - category: Connectors / Ingestion / Lineage url: /connectors/ingestion/lineage - category: Connectors / Ingestion / Lineage / Edit Data Lineage Manually diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/index.md index 842bbef2042..21b50fa6e74 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/athena name="Athena" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "Tags", "dbt", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "Tags", "dbt", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Stored Procedures"] / %} @@ -24,7 +24,8 @@ Configure and schedule Athena metadata and profiler workflows from the OpenMetad - [Troubleshooting](/connectors/database/athena/troubleshooting) - [Workflow Deployment Error](#workflow-deployment-error) - [Related](#related) -- [Reverse Metadata](#reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-link.md" collate: true /%} + {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/athena/yaml"} /%} ## Requirements @@ -237,38 +238,7 @@ Find more information about [Source Identity](https://docs.aws.amazon.com/STS/la {% /stepsContainer %} + +{% partial file="/v1.7/connectors/database/athena/reverse-metadata.md" collate: true /%} + {% partial file="/v1.7/connectors/database/related.md" /%} - -## Reverse Metadata - -{% note %} -This feature is specific to Collate and requires the Collate Enterprise License. -{% /note %} - -### Description Management - -Athena supports description updates at the following level: -- Table level - -### Owner Management - -❌ Owner management is not supported for Athena. - -### Tag Management - -❌ Tag management is not supported for Athena. - -### Custom SQL Template - -Athena supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. - -Here are examples of custom SQL queries for metadata changes: - -```sql --- Update table description -ALTER TABLE {schema}.{table} SET TBLPROPERTIES ('comment' = {description}); -``` - -The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). - -For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/yaml.md index b3ea0c9b4e1..f9fecb86c3d 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/athena/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/athena/yaml name="Athena" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "Tags", "dbt", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "Tags", "dbt", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Stored Procedures"] / %} @@ -22,7 +22,7 @@ Configure and schedule Athena metadata and profiler workflows from the OpenMetad - [Data Profiler](#data-profiler) - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) -- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-workflow-link.md" collate: true /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/index.md index c7de4bd2152..300bb50a6e8 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/bigquery name="BigQuery" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Stored Procedures", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners"] / %} @@ -24,10 +24,8 @@ Configure and schedule BigQuery metadata and profiler workflows from the OpenMet - [Data Quality](/how-to-guides/data-quality-observability/quality/configure) - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) -- [Reverse Metadata Ingestion](#reverse-metadata-ingestion) - [Troubleshooting](/connectors/database/bigquery/troubleshooting) -- [Reverse Metadata](#reverse-metadata) - +{% partial file="/v1.7/connectors/reverse-metadata-link.md" collate: true /%} {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/bigquery/yaml"} /%} @@ -164,41 +162,6 @@ the GCP credentials empty. This is why they are not marked as required. We support cross-project lineage, but the data must be ingested within a single service. This means you need to perform lineage ingestion for just one service while including multiple projects. +{% partial file="/v1.7/connectors/database/bigquery/reverse-metadata.md" collate: true /%} + {% partial file="/v1.7/connectors/database/related.md" /%} - -## Reverse Metadata - -{% note %} -This feature is specific to Collate and requires the Collate Enterprise License. -{% /note %} - -### Description Management - -BigQuery supports description updates at the following levels: -- Schema level -- Table level - -### Owner Management - -❌ Owner management is not supported for BigQuery. - -### Tag Management - -BigQuery supports tag management at the following levels: -- Schema level -- Table level - -### Custom SQL Template - -BigQuery supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. - -Here are examples of custom SQL queries for metadata changes: - -```sql --- Update table labels -ALTER TABLE `{database}.{schema}.{table}` SET OPTIONS (labels = {tags}); -``` - -The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). - -For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/yaml.md index cae61aa8344..257c5ac5f9a 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/bigquery/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/bigquery/yaml name="BigQuery" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Stored Procedures", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners"] / %} @@ -23,7 +23,7 @@ Configure and schedule BigQuery metadata and profiler workflows from the OpenMet - [Data Profiler](#data-profiler) - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) -- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-workflow-link.md" collate: true /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/index.md index 8b5e86e98d5..e8c364e3b9f 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/clickhouse name="Clickhouse" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Tags", "Stored Procedures"] / %} @@ -26,7 +26,7 @@ Configure and schedule Clickhouse metadata and profiler workflows from the OpenM - [Troubleshooting](/connectors/database/clickhouse/troubleshooting) - [Workflow Deployment Error](#workflow-deployment-error) - [Related](#related) -- [Reverse Metadata](#reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-link.md" collate: true /%} {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/clickhouse/yaml"} /%} @@ -94,39 +94,6 @@ For the usage and lineage workflow, the user will need `SELECT` privilege. You c {% /stepsContainer %} +{% partial file="/v1.7/connectors/database/clickhouse/reverse-metadata.md" collate: true /%} + {% partial file="/v1.7/connectors/database/related.md" /%} - -## Reverse Metadata - -{% note %} -This feature is specific to Collate and requires the Collate Enterprise License. -{% /note %} - -### Description Management - -Clickhouse supports description updates at the following levels: -- Table level -- Column level - -### Owner Management - -❌ Owner management is not supported for Clickhouse. - -### Tag Management - -❌ Tag management is not supported for Clickhouse. - -### Custom SQL Template - -Clickhouse supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. - -Here are examples of custom SQL queries for metadata changes: - -```sql --- Update column description -ALTER TABLE `{schema}`.{table} MODIFY COLUMN {column} COMMENT {description}; -``` - -The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). - -For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/yaml.md index fd751c1156c..567c652e152 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/clickhouse/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/clickhouse/yaml name="Clickhouse" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Tags", "Stored Procedures"] / %} @@ -22,7 +22,7 @@ Configure and schedule Clickhouse metadata and profiler workflows from the OpenM - [Data Profiler](#data-profiler) - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) -- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-workflow-link.md" collate: true /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/index.md index 4859219e5dd..a8c51b9253f 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/databricks name="Databricks" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Stored Procedures"] / %} @@ -27,8 +27,8 @@ Configure and schedule Databricks metadata and profiler workflows from the OpenM - [Data Quality](/how-to-guides/data-quality-observability/quality) - [Lineage](/how-to-guides/data-lineage/workflow) - [dbt Integration](/connectors/ingestion/workflows/dbt) -- [Reverse Metadata](#reverse-metadata) - [Troubleshooting](/connectors/database/databricks/troubleshooting) +{% partial file="/v1.7/connectors/reverse-metadata-link.md" collate: true /%} {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/databricks/yaml"} /%} @@ -74,48 +74,6 @@ If you are using unity catalog in Databricks, then checkout the [Unity Catalog]( {% /stepsContainer %} +{% partial file="/v1.7/connectors/database/databricks/reverse-metadata.md" collate: true /%} + {% partial file="/v1.7/connectors/database/related.md" /%} - -## Reverse Metadata - -{% note %} -This feature is specific to Collate and requires the Collate Enterprise License. -{% /note %} - -### Description Management - -Databricks supports description updates at all levels: -- Database level -- Schema level -- Table level -- Column level - -### Owner Management - -Databricks supports owner management at the following levels: -- Database level -- Schema level -- Table level - -### Tag Management - -Databricks supports tag management at all levels: -- Database level -- Schema level -- Table level -- Column level - -### Custom SQL Template - -Databricks supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. - -Here are examples of custom SQL queries for metadata changes: - -```sql --- Set table tags -ALTER TABLE {database}.{schema}.{table} SET TAGS {tags}; -``` - -The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). - -For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/yaml.md index bdef8bf16ef..4fca2e4f102 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/databricks/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/databricks/yaml name="Databricks" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Lineage", "Column-level Lineage", "Data Profiler", "Data Quality", "dbt", "Tags", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Stored Procedures"] / %} @@ -26,7 +26,7 @@ Configure and schedule Databricks metadata and profiler workflows from the OpenM - [Data Profiler](#data-profiler) - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) -- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-workflow-link.md" collate: true /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} ## Requirements diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/index.md index 5ffa5659d5f..703a9b79100 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/mssql name="MSSQL" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Tags", "SSIS packages"] / %} @@ -23,8 +23,8 @@ Configure and schedule MSSQL metadata and profiler workflows from the OpenMetada - [Data Quality](/how-to-guides/data-quality-observability/quality) - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) -- [Reverse Metadata](#reverse-metadata) - [Troubleshooting](/connectors/database/mssql/troubleshooting) +{% partial file="/v1.7/connectors/reverse-metadata-link.md" collate: true /%} {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/mssql/yaml"} /%} @@ -122,62 +122,6 @@ For details step please refer to this [link](https://docs.microsoft.com/en-us/sq {% /stepsContainer %} +{% partial file="/v1.7/connectors/database/mssql/reverse-metadata.md" collate: true /%} + {% partial file="/v1.7/connectors/database/related.md" /%} - -## Reverse Metadata - -{% note %} -This feature is specific to Collate and requires the Collate Enterprise License. -{% /note %} - -### Description Management - -MSSQL supports description updates at the following levels: -- Schema level -- Table level -- Column level - -### Owner Management - -MSSQL supports owner management at the following levels: -- Database level -- Schema level - -### Tag Management - -❌ Tag management is not supported for MSSQL. - -### Custom SQL Template - -MSSQL supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. - -Here are examples of custom SQL queries for metadata changes: - -```sql --- Update schema description -IF NOT EXISTS (SELECT 1 - FROM {database}.sys.extended_properties - WHERE name = N'MS_Description' And - major_id = SCHEMA_ID('{schema}') AND - minor_id = 0 - ) -BEGIN - EXEC {database}.{schema}.sp_addextendedproperty - @name = N'MS_Description', - @value = N{description}, - @level0type = N'SCHEMA', - @level0name = N'{schema}' -END -ELSE -BEGIN - EXEC {database}.{schema}.sp_updateextendedproperty - @name = N'MS_Description', - @value = N{description}, - @level0type = N'SCHEMA', - @level0name = N'{schema}' -END; -``` - -The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). - -For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/yaml.md index fbef5c6defb..8a3d2415fed 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mssql/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/mssql/yaml name="MSSQL" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Tags", "SSIS packages"] / %} @@ -22,7 +22,7 @@ Configure and schedule MSSQL metadata and profiler workflows from the OpenMetada - [Data Profiler](#data-profiler) - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) -- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-workflow-link.md" collate: true /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} ## Requirements diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/index.md index 4bf65d4424c..cb6c576de10 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/mysql name="MySQL" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Data Profiler", "Data Quality", "dbt", "View Lineage", "View Column-level Lineage", "Query Usage", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Data Profiler", "Data Quality", "dbt", "View Lineage", "View Column-level Lineage", "Query Usage", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Tags", "Stored Procedures"] / %} @@ -22,8 +22,8 @@ Configure and schedule MySQL metadata and profiler workflows from the OpenMetada - [dbt Integration](/connectors/ingestion/workflows/dbt) - [Enable Security](#securing-mysql-connection-with-ssl-in-openmetadata) - [Data Lineage](/how-to-guides/data-lineage/workflow) -- [Reverse Metadata](#reverse-metadata) - [Troubleshooting](/connectors/database/mysql/troubleshooting) +{% partial file="/v1.7/connectors/reverse-metadata-link.md" collate: true /%} {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/mysql/yaml"} /%} @@ -245,43 +245,6 @@ To establish secure connections between OpenMetadata and MySQL, navigate to the height="450px" caption="SSL Configuration" /%} +{% partial file="/v1.7/connectors/database/mysql/reverse-metadata.md" collate: true /%} + {% partial file="/v1.7/connectors/database/related.md" /%} - -## Reverse Metadata - -{% note %} -This feature is specific to Collate and requires the Collate Enterprise License. -{% /note %} - -### Description Management - -MySQL supports description updates at the following level: -- Table level - -### Owner Management - -Owner management is not supported for MySQL. - -### Tag Management - -❌ Tag management is not supported for MySQL. - -### Custom SQL Template - -MySQL supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. - -Here are examples of custom SQL queries for metadata changes: - -```sql --- Update table description -ALTER TABLE {schema}.{table} COMMENT = {description}; -``` - -The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). - -For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). - -### Requirements for Reverse Metadata - -In addition to the basic ingestion requirements, for reverse metadata ingestion the user needs: -- ` \ No newline at end of file diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md index 1c5a9ba0d0c..099f9bbdd6d 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/mysql/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/mysql/yaml name="MySQL" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Data Profiler", "Data Quality", "dbt", "View Lineage", "View Column-level Lineage", "Query Usage", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Data Profiler", "Data Quality", "dbt", "View Lineage", "View Column-level Lineage", "Query Usage", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Tags", "Stored Procedures"] / %} @@ -22,7 +22,7 @@ Configure and schedule MySQL metadata and profiler workflows from the OpenMetada - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) - [Enable Security](#securing-mysql-connection-with-ssl-in-openmetadata) -- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-workflow-link.md" collate: true /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} ## Requirements diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/index.md index df3ed718a53..2c56c8d66e8 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/oracle name="Oracle" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Tags"] / %} @@ -21,8 +21,8 @@ Configure and schedule Oracle metadata and profiler workflows from the OpenMetad - [Data Quality](/how-to-guides/data-quality-observability/quality) - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) -- [Reverse Metadata](#reverse-metadata) - [Troubleshooting](/connectors/database/oracle/troubleshooting) +{% partial file="/v1.7/connectors/reverse-metadata-link.md" collate: true /%} {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/oracle/yaml"} /%} @@ -132,39 +132,6 @@ there is no routine out of the box in Oracle to grant SELECT to a full schema. {% /stepsContainer %} +{% partial file="/v1.7/connectors/database/oracle/reverse-metadata.md" collate: true /%} + {% partial file="/v1.7/connectors/database/related.md" /%} - -## Reverse Metadata - -{% note %} -This feature is specific to Collate and requires the Collate Enterprise License. -{% /note %} - -### Description Management - -Oracle supports description updates at the following levels: -- Table level -- Column level - -### Owner Management - -❌ Owner management is not supported for Oracle. - -### Tag Management - -❌ Tag management is not supported for Oracle. - -### Custom SQL Template - -Oracle supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. - -Here are examples of custom SQL queries for metadata changes: - -```sql --- Update column description -COMMENT ON COLUMN {schema}.{table}.{column} IS {description} -``` - -The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). - -For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/yaml.md index 8e89b7c127f..a676b82abeb 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/oracle/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/oracle/yaml name="Oracle" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Tags"] / %} @@ -21,7 +21,7 @@ Configure and schedule Oracle metadata and profiler workflows from the OpenMetad - [Data Quality](#data-quality) - [Lineage](#lineage) - [dbt Integration](#dbt-integration) -- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-workflow-link.md" collate: true /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} ## Requirements diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/index.md index e7433f81048..467f1a38eb6 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/postgres name="PostgreSQL" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Owners", "Tags", "Stored Procedures", "Sample Data", "Stored Procedures Lineage", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Owners", "Tags", "Stored Procedures", "Sample Data", "Stored Procedures Lineage", "Reverse Metadata (Collate Only)"] unavailableFeatures=[] / %} @@ -23,8 +23,8 @@ Configure and schedule PostgreSQL metadata and profiler workflows from the OpenM - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) - [Enable Security](#securing-postgres-connection-with-ssl-in-openmetadata) -- [Reverse Metadata](#reverse-metadata) - [Troubleshooting](/connectors/database/postgres/troubleshooting) +{% partial file="/v1.7/connectors/reverse-metadata-link.md" collate: true /%} {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/postgres/yaml"} /%} @@ -216,54 +216,6 @@ For IAM authentication, it is recommended to choose the `allow` mode or another height="450px" caption="SSL Configuration" /%} +{% partial file="/v1.7/connectors/database/postgres/reverse-metadata.md" collate: true /%} + {% partial file="/v1.7/connectors/database/related.md" /%} - -## Reverse Metadata - -{% note %} -This feature is specific to Collate and requires the Collate Enterprise License. -{% /note %} - -### Description Management - -PostgreSQL supports description updates at all levels: -- Database level -- Schema level -- Table level -- Column level - -### Owner Management - -PostgreSQL supports owner management at the following levels: -- Database level -- Schema level -- Table level - -### Tag Management - -❌ Tag management is not supported for PostgreSQL. - -### Custom SQL Template - -PostgreSQL supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. - -Here are examples of custom SQL queries for metadata changes: - -```sql --- Update database owner -ALTER DATABASE {database} OWNER TO {owner}; -``` - -The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). - -For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). - -### Requirements for Reverse Metadata - -In addition to the basic ingestion requirements, for reverse metadata ingestion the user needs: -- `SUPERUSER` privilege to the user - -```sql --- Grant superuser privilege to the user -ALTER ROLE USER_NAME SUPERUSER; -``` diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/yaml.md index b76017cb249..13d7d33d1da 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/postgres/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/postgres/yaml name="PostgreSQL" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Owners", "Tags", "Sample Data", "Stored Procedures", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Owners", "Tags", "Sample Data", "Stored Procedures", "Reverse Metadata (Collate Only)"] unavailableFeatures=[] / %} @@ -23,7 +23,7 @@ Configure and schedule PostgreSQL metadata and profiler workflows from the OpenM - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) - [Enable Security](#securing-postgres-connection-with-ssl-in-openmetadata) -- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-workflow-link.md" collate: true /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} ## Requirements diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/index.md index 411243e6c1c..fd90063a95c 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/redshift name="Redshift" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Tags"] / %} @@ -24,8 +24,8 @@ Configure and schedule Redshift metadata and profiler workflows from the OpenMet - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) - [Enable Security](#securing-redshift-connection-with-ssl-in-openmetadata) -- [Reverse Metadata](#reverse-metadata) -troubleshooting) +- [Troubleshooting](/connectors/database/redshift/troubleshooting) +{% partial file="/v1.7/connectors/reverse-metadata-link.md" collate: true /%} {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/redshift/yaml"} /%} @@ -129,44 +129,6 @@ Under `Advanced Config`, specify the SSL mode appropriate for your connection, s height="450px" caption="SSL Configuration" /%} +{% partial file="/v1.7/connectors/database/redshift/reverse-metadata.md" collate: true /%} + {% partial file="/v1.7/connectors/database/related.md" /%} - -## Reverse Metadata - -{% note %} -This feature is specific to Collate and requires the Collate Enterprise License. -{% /note %} - -### Description Management - -Redshift supports description updates at all levels: -- Database level -- Schema level -- Table level -- Column level - -### Owner Management - -Redshift supports owner management at the following levels: -- Database level -- Schema level -- Table level - -### Tag Management - -❌ Tag management is not supported for Redshift. - -### Custom SQL Template - -Redshift supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. - -Here are examples of custom SQL queries for metadata changes: - -```sql --- Update column description -COMMENT ON COLUMN {database}.{schema}.{table}.{column} IS {description}; -``` - -The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). - -For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/yaml.md index 74e54a06a9a..c2206ecfe86 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/redshift/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/redshift/yaml name="Redshift" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "dbt", "Lineage", "Column-level Lineage", "Stored Procedures", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Tags"] / %} @@ -24,7 +24,7 @@ Configure and schedule Redshift metadata and profiler workflows from the OpenMet - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) - [Enable Security](#securing-redshift-connection-with-ssl-in-openmetadata) -- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-workflow-link.md" collate: true /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/index.md index 4c2588ab79c..c0a997564a0 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/snowflake name="Snowflake" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Stored Procedures", "Tags", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Stored Procedures", "Tags", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners"] / %} @@ -24,8 +24,8 @@ Configure and schedule Snowflake metadata and profiler workflows from the OpenMe - [Data Quality](/how-to-guides/data-quality-observability/quality) - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) -- [Reverse Metadata](#reverse-metadata) - [Troubleshooting](/connectors/database/snowflake/troubleshooting) +{% partial file="/v1.7/connectors/reverse-metadata-link.md" collate: true /%} {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/snowflake/yaml"} /%} @@ -149,81 +149,6 @@ GRANT APPLY TAG TO ROLE NEW_ROLE; Depending on your view ddl you can grant the relevant privileged as per above queries. +{% partial file="/v1.7/connectors/database/snowflake/reverse-metadata.md" collate: true /%} + {% partial file="/v1.7/connectors/database/related.md" /%} - - -## Reverse Metadata - -{% note %} -This feature is specific to Collate and requires the Collate Enterprise License. -{% /note %} - -### Description Management - -Snowflake supports full description updates at all levels: -- Database level -- Schema level -- Table level -- Column level - -### Owner Management - -❌ Owner management is not supported for Snowflake. - -### Tag Management - -Snowflake supports tag management at the following levels: -- Schema level -- Table level -- Column level - -Additionally, Snowflake supports automated masking policy application based on tags. For example, when you apply a `Sensitive` tag to a column in OpenMetadata, the corresponding masking policy will be automatically applied to that column in Snowflake. - -{% note %} -While OpenMetadata differentiates between "classification" values and "tag" values, Snowflake's masking policies only support actions based on classification values (tag keys). Therefore, it is advised not to use PII.None along with tag masking policies. -{% /note %} - -```sql --- Create masking policy -CREATE MASKING POLICY SENSITIVE_DATA AS (VAL STRING) RETURNS STRING -> CASE WHEN VAL IS NOT NULL THEN '**********' ELSE NULL END; - --- Apply masking policy to PII.Sensitive tag -ALTER TAG PII SET MASKING POLICY SENSITIVE_DATA; -``` - -### Custom SQL Template - -Snowflake supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. - -Here are examples of custom SQL queries for metadata changes: - -```sql --- Set schema tag -ALTER SCHEMA {database}.{schema} SET TAG {database}.{schema}.{tag_key} = '{tag_value}'; -``` - -The list of variables for custom SQL can be found [here](/connectors/ingestion/workflows/reverse-metadata#custom-sql-template). - -For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). - -### Requirements for Reverse Metadata - -In addition to the basic ingestion requirements, for reverse metadata ingestion the user needs: -- `ACCOUNTADMIN` role or a role with similar privileges to modify descriptions and tags -- Access to `snowflake.account_usage.tag_references` for tag management - -```sql --- Grant modify database privileges to the role -GRANT MODIFY ON DATABASE DATABASE_NAME TO ROLE NEW_ROLE; - --- Grant ownership on schema to the role -GRANT OWNERSHIP ON SCHEMA DATABASE_NAME.SCHEMA_NAME TO ROLE NEW_ROLE REVOKE CURRENT GRANTS; - --- Grant ownership on table to the role -GRANT OWNERSHIP ON TABLE DATABASE_NAME.SCHEMA_NAME.TABLE_NAME TO ROLE NEW_ROLE REVOKE CURRENT GRANTS; - --- Grant imported privileges on snowflake database to the role -GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE NEW_ROLE; -``` - -For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/yaml.md index 020a421a8a2..ee78c3ae1a1 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/snowflake/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/snowflake/yaml name="Snowflake" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Stored Procedures", "Tags", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Stored Procedures", "Tags", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners"] / %} @@ -23,7 +23,7 @@ Configure and schedule Snowflake metadata and profiler workflows from the OpenMe - [Data Profiler](#data-profiler) - [Data Quality](#data-quality) - [dbt Integration](#dbt-integration) -- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-workflow-link.md" collate: true /%} {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/index.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/index.md index a382713fc25..c8477375549 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/index.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/index.md @@ -7,7 +7,7 @@ slug: /connectors/database/unity-catalog name="Unity Catalog" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Tags", "Stored Procedures"] / %} @@ -20,8 +20,8 @@ Configure and schedule Unity Catalog metadata workflow from the OpenMetadata UI: - [Data Quality](/how-to-guides/data-quality-observability/quality) - [Lineage](/connectors/ingestion/lineage) - [dbt Integration](/connectors/ingestion/workflows/dbt) -- [Reverse Metadata](#reverse-metadata) - [Troubleshooting](/connectors/database/unity-catalog/troubleshooting) +{% partial file="/v1.7/connectors/reverse-metadata-link.md" collate: true /%} {% partial file="/v1.7/connectors/ingestion-modes-tiles.md" variables={yamlPath: "/connectors/database/unity-catalog/yaml"} /%} @@ -63,41 +63,6 @@ Configure and schedule Unity Catalog metadata workflow from the OpenMetadata UI: {% /stepsContainer %} +{% partial file="/v1.7/connectors/database/unitycatalog/reverse-metadata.md" collate: true /%} + {% partial file="/v1.7/connectors/database/related.md" /%} - -## Reverse Metadata - -{% note %} -This feature is specific to Collate and requires the Collate Enterprise License. -{% /note %} - -### Description Management - -Unity Catalog supports description updates at all levels: -- Database level -- Schema level -- Table level -- Column level - -### Owner Management - -Unity Catalog supports owner management at the following levels: -- Database level -- Schema level -- Table level - -### Tag Management - -Unity Catalog supports tag management at all levels: -- Database level -- Schema level -- Table level -- Column level - -### Custom SQL Template - -Unity Catalog supports custom SQL templates for metadata changes. The template is interpreted using python f-strings. - -Examples will be provided in a future update. - -For more details about reverse metadata ingestion, visit our [Reverse Metadata Documentation](/connectors/ingestion/workflows/reverse-metadata). diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/yaml.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/yaml.md index 98d8300fa43..e7de43a69e4 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/yaml.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/database/unity-catalog/yaml.md @@ -7,7 +7,7 @@ slug: /connectors/database/unity-catalog/yaml name="Unity Catalog" stage="PROD" platform="OpenMetadata" -availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Sample Data", "Reverse Metadata Ingestion"] +availableFeatures=["Metadata", "Query Usage", "Data Profiler", "Data Quality", "Lineage", "Column-level Lineage", "dbt", "Sample Data", "Reverse Metadata (Collate Only)"] unavailableFeatures=["Owners", "Tags", "Stored Procedures"] / %} @@ -20,7 +20,8 @@ Configure and schedule Unity Catalog metadata workflow from the OpenMetadata UI: - [Query Usage](#query-usage) - [Lineage](#lineage) - [dbt Integration](#dbt-integration) -- [Reverse Metadata Ingestion](/connectors/ingestion/workflows/reverse-metadata) +{% partial file="/v1.7/connectors/reverse-metadata-workflow-link.md" collate: true /%} + {% partial file="/v1.7/connectors/external-ingestion-deployment.md" /%} ## Requirements diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/ingestion/workflows/reverse-metadata.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/ingestion/workflows/reverse-metadata.md index fd849ca2100..e15fd8059bd 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/ingestion/workflows/reverse-metadata.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/ingestion/workflows/reverse-metadata.md @@ -1,12 +1,11 @@ --- title: Reverse Metadata Ingestion Collate | Feature & Configuration slug: /connectors/ingestion/workflows/reverse-metadata +collate: true --- # Reverse Metadata -{% note %} -This feature is specific to Collate and requires the Collate Enterprise License. -{% /note %} + Reverse Metadata is an advanced feature in OpenMetadata that facilitates bi-directional synchronization between OpenMetadata and the source database systems. While standard ingestion pulls metadata into OpenMetadata, reverse ingestion enables pushing metadata changes made within OpenMetadata back to the source systems. This ensures consistency and alignment across the entire data infrastructure. @@ -19,17 +18,17 @@ Reverse Metadata uses the existing service connection configuration provided dur | Database | Update Description | Update Tags | Update Owners | Custom SQL Support | Documentation | |----------------|-------------------|-------------|---------------|-------------------|---------------| -| Athena | ✅ (Table) | ❌ | ❌ | ✅ | [Link](/connectors/database/athena#reverse-metadata-ingestion) | -| BigQuery | ✅ (Schema, Table)| ✅ (Schema, Table) | ❌ | ✅ | [Link](/connectors/database/bigquery#reverse-metadata-ingestion) | -| Clickhouse | ✅ (Table, Column)| ❌ | ❌ | ✅ | [Link](/connectors/database/clickhouse#reverse-metadata-ingestion) | -| Databricks | ✅ | ✅ | ✅ (Database, Schema, Table) | ✅ | [Link](/connectors/database/databricks#reverse-metadata-ingestion) | -| MSSQL | ✅ (Schema, Table, Column) | ❌ | ✅ (Database, Schema) | ✅ | [Link](/connectors/database/mssql#reverse-metadata-ingestion) | -| MySQL | ✅ (Table) | ❌ | ❌ | ✅ | [Link](/connectors/database/mysql#reverse-metadata-ingestion) | -| Oracle | ✅ (Table, Column)| ❌ | ❌ | ✅ | [Link](/connectors/database/oracle#reverse-metadata-ingestion) | -| PostgreSQL | ✅ | ❌ | ✅ (Database, Schema, Table) | ✅ | [Link](/connectors/database/postgres#reverse-metadata-ingestion) | -| Redshift | ✅ | ❌ | ✅ (Database, Schema, Table) | ✅ | [Link](/connectors/database/redshift#reverse-metadata-ingestion) | -| Snowflake | ✅ | ✅ (Schema, Table, Column) | ❌ | ✅ | [Link](/connectors/database/snowflake#reverse-metadata-ingestion) | -| Unity Catalog | ✅ | ✅ | ✅ (Database, Schema, Table) | ✅ | [Link](/connectors/database/unity-catalog#reverse-metadata-ingestion) | +| Athena | ✅ (Table) | ❌ | ❌ | ✅ | [Link](/connectors/database/athena#reverse-metadata) | +| BigQuery | ✅ (Schema, Table)| ✅ (Schema, Table) | ❌ | ✅ | [Link](/connectors/database/bigquery#reverse-metadata) | +| Clickhouse | ✅ (Table, Column)| ❌ | ❌ | ✅ | [Link](/connectors/database/clickhouse#reverse-metadata) | +| Databricks | ✅ | ✅ | ✅ (Database, Schema, Table) | ✅ | [Link](/connectors/database/databricks#reverse-metadata) | +| MSSQL | ✅ (Schema, Table, Column) | ❌ | ✅ (Database, Schema) | ✅ | [Link](/connectors/database/mssql#reverse-metadata) | +| MySQL | ✅ (Table) | ❌ | ❌ | ✅ | [Link](/connectors/database/mysql#reverse-metadata) | +| Oracle | ✅ (Table, Column)| ❌ | ❌ | ✅ | [Link](/connectors/database/oracle#reverse-metadata) | +| PostgreSQL | ✅ | ❌ | ✅ (Database, Schema, Table) | ✅ | [Link](/connectors/database/postgres#reverse-metadata) | +| Redshift | ✅ | ❌ | ✅ (Database, Schema, Table) | ✅ | [Link](/connectors/database/redshift#reverse-metadata) | +| Snowflake | ✅ | ✅ (Schema, Table, Column) | ❌ | ✅ | [Link](/connectors/database/snowflake#reverse-metadata) | +| Unity Catalog | ✅ | ✅ | ✅ (Database, Schema, Table) | ✅ | [Link](/connectors/database/unity-catalog#reverse-metadata) | ## Key Features diff --git a/openmetadata-docs/content/v1.7.x-SNAPSHOT/menu.md b/openmetadata-docs/content/v1.7.x-SNAPSHOT/menu.md index b2a6a861dfb..737599dca82 100644 --- a/openmetadata-docs/content/v1.7.x-SNAPSHOT/menu.md +++ b/openmetadata-docs/content/v1.7.x-SNAPSHOT/menu.md @@ -808,9 +808,6 @@ site_menu: - category: Connectors / Ingestion / Workflows/ Metadata / Multithreading url: /connectors/ingestion/workflows/metadata/multithreading - - category: Connectors / Ingestion / Workflows / Reverse Metadata Ingestion - url: /connectors/ingestion/workflows/reverse-metadata - - category: Connectors / Ingestion / Workflows / Usage url: /connectors/ingestion/workflows/usage - category: Connectors / Ingestion / Workflows / Usage / Usage Workflow Through Query Logs