From 980e722e24547c095e7ae09f0ac6b76c07eb8ddf Mon Sep 17 00:00:00 2001 From: Rounak Dhillon <162090200+RounakDhillon@users.noreply.github.com> Date: Fri, 30 May 2025 08:43:57 +0530 Subject: [PATCH] Doc: How Collate AI Enhances Autopilot Workflows Doc Updation (#21445) --- .../collate-ai-autopilot-workflows.md | 57 +++++++++++++++++++ .../content/v1.7.x/collate-ai/index.md | 2 + .../content/v1.7.x/collate-menu.md | 2 + .../collate-ai-autopilot-workflows.md | 57 +++++++++++++++++++ .../v1.8.x-SNAPSHOT/collate-ai/index.md | 2 + .../content/v1.8.x-SNAPSHOT/collate-menu.md | 2 + 6 files changed, 122 insertions(+) create mode 100644 openmetadata-docs/content/v1.7.x/collate-ai/collate-ai-autopilot-workflows.md create mode 100644 openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-ai/collate-ai-autopilot-workflows.md diff --git a/openmetadata-docs/content/v1.7.x/collate-ai/collate-ai-autopilot-workflows.md b/openmetadata-docs/content/v1.7.x/collate-ai/collate-ai-autopilot-workflows.md new file mode 100644 index 00000000000..0c2efef6c4b --- /dev/null +++ b/openmetadata-docs/content/v1.7.x/collate-ai/collate-ai-autopilot-workflows.md @@ -0,0 +1,57 @@ +--- +title: Streamlining Metadata with Collate AI and Autopilot Integration +slug: /collateai/collate-ai-autopilot-workflows +collate: true +--- + +# How Collate AI Enhances Autopilot Workflows + +Collate AI seamlessly integrates with the Autopilot application in Collate to automate and enhance metadata management processes. This integration streamlines the onboarding of new data services and enriches metadata through AI-driven automation. + +## How Collate AI Integrates with Autopilot + +{% youtube videoId="xV-WTva8vG8" start="0:00" end="4:20" width="800px" height="450px" /%} + +### Automated Metadata Ingestion with Autopilot + +When a new data service is added, Autopilot automatically initiates agents that manage ingestion workflows. These workflows extract: + +- Schemas +- Lineage +- Usage +- Profiling information + +This forms the foundation for further AI-driven enhancements. + +### AI Agents Enhancing Metadata + +Post-ingestion, Collate AI deploys specialized agents to enrich metadata: + +- **Documentation Agent** + Generates descriptive metadata for tables and columns, improving understanding and discoverability. + +- **Tiering Agent** + Analyzes data usage and lineage to assign business-criticality tiers to data assets. + +- **Data Quality Agent** + Creates data quality tests based on table constraints and patterns, ensuring reliability. + +## Insights and Monitoring + +The outcomes of these processes are available in the **Service Insights dashboard**, offering key performance indicators such as: + +- Percentage of assets with generated descriptions +- Distribution of data assets across different tiers +- Data quality health metrics +- Identification of assets containing PII + +## Benefits of the Integration + +- **Efficiency** + Reduces manual effort in metadata documentation and quality assurance. + +- **Consistency** + Ensures uniform metadata standards across the organization. + +- **Visibility** + Provides clear insights into data assets, aiding governance and compliance. diff --git a/openmetadata-docs/content/v1.7.x/collate-ai/index.md b/openmetadata-docs/content/v1.7.x/collate-ai/index.md index 3641b41cdee..7df392bd0a5 100644 --- a/openmetadata-docs/content/v1.7.x/collate-ai/index.md +++ b/openmetadata-docs/content/v1.7.x/collate-ai/index.md @@ -15,6 +15,8 @@ Collate AI is an AI-powered tool within Collate that simplifies and enhances met - **Automated Data Documentation**: Automatically generates metadata descriptions for tables and columns, saving time and effort for data owners and stewards. - **Natural Language SQL Query Generation**: Allows users to interact with Collate AI through a chatbot to generate SQL queries by simply asking questions in plain English. - **SQL Query Optimization and Fixing**: Capable of optimizing and troubleshooting SQL queries to improve their performance and efficiency. +- **AI-Driven Test Automation**: Automatically suggests and deploys relevant data quality tests based on table constraints and similarity with other assets, streamlining your data validation efforts. +- **Automated Tier Classification**: Uses data lineage and usage analytics to intelligently assign Tier levels, helping teams prioritize and govern critical data assets. ## Why Collate AI is Useful diff --git a/openmetadata-docs/content/v1.7.x/collate-menu.md b/openmetadata-docs/content/v1.7.x/collate-menu.md index 75fa3438603..a513ca73423 100644 --- a/openmetadata-docs/content/v1.7.x/collate-menu.md +++ b/openmetadata-docs/content/v1.7.x/collate-menu.md @@ -719,6 +719,8 @@ site_menu: - category: Collate AI url: /collateai + - category: Collate AI / Autopilot + url: /collateai/collate-ai-autopilot-workflows - category: Collate AI / How to Use Collate AI url: /collateai/how-to-use-collate-ai - category: Collate AI / Collate AI Tier Agent diff --git a/openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-ai/collate-ai-autopilot-workflows.md b/openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-ai/collate-ai-autopilot-workflows.md new file mode 100644 index 00000000000..0c2efef6c4b --- /dev/null +++ b/openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-ai/collate-ai-autopilot-workflows.md @@ -0,0 +1,57 @@ +--- +title: Streamlining Metadata with Collate AI and Autopilot Integration +slug: /collateai/collate-ai-autopilot-workflows +collate: true +--- + +# How Collate AI Enhances Autopilot Workflows + +Collate AI seamlessly integrates with the Autopilot application in Collate to automate and enhance metadata management processes. This integration streamlines the onboarding of new data services and enriches metadata through AI-driven automation. + +## How Collate AI Integrates with Autopilot + +{% youtube videoId="xV-WTva8vG8" start="0:00" end="4:20" width="800px" height="450px" /%} + +### Automated Metadata Ingestion with Autopilot + +When a new data service is added, Autopilot automatically initiates agents that manage ingestion workflows. These workflows extract: + +- Schemas +- Lineage +- Usage +- Profiling information + +This forms the foundation for further AI-driven enhancements. + +### AI Agents Enhancing Metadata + +Post-ingestion, Collate AI deploys specialized agents to enrich metadata: + +- **Documentation Agent** + Generates descriptive metadata for tables and columns, improving understanding and discoverability. + +- **Tiering Agent** + Analyzes data usage and lineage to assign business-criticality tiers to data assets. + +- **Data Quality Agent** + Creates data quality tests based on table constraints and patterns, ensuring reliability. + +## Insights and Monitoring + +The outcomes of these processes are available in the **Service Insights dashboard**, offering key performance indicators such as: + +- Percentage of assets with generated descriptions +- Distribution of data assets across different tiers +- Data quality health metrics +- Identification of assets containing PII + +## Benefits of the Integration + +- **Efficiency** + Reduces manual effort in metadata documentation and quality assurance. + +- **Consistency** + Ensures uniform metadata standards across the organization. + +- **Visibility** + Provides clear insights into data assets, aiding governance and compliance. diff --git a/openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-ai/index.md b/openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-ai/index.md index 882616f7095..77e150e8441 100644 --- a/openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-ai/index.md +++ b/openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-ai/index.md @@ -15,6 +15,8 @@ Collate AI is an AI-powered tool within Collate that simplifies and enhances met - **Automated Data Documentation**: Automatically generates metadata descriptions for tables and columns, saving time and effort for data owners and stewards. - **Natural Language SQL Query Generation**: Allows users to interact with Collate AI through a chatbot to generate SQL queries by simply asking questions in plain English. - **SQL Query Optimization and Fixing**: Capable of optimizing and troubleshooting SQL queries to improve their performance and efficiency. +- **AI-Driven Test Automation**: Automatically suggests and deploys relevant data quality tests based on table constraints and similarity with other assets, streamlining your data validation efforts. +- **Automated Tier Classification**: Uses data lineage and usage analytics to intelligently assign Tier levels, helping teams prioritize and govern critical data assets. ## Why Collate AI is Useful diff --git a/openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-menu.md b/openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-menu.md index 75fa3438603..a513ca73423 100644 --- a/openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-menu.md +++ b/openmetadata-docs/content/v1.8.x-SNAPSHOT/collate-menu.md @@ -719,6 +719,8 @@ site_menu: - category: Collate AI url: /collateai + - category: Collate AI / Autopilot + url: /collateai/collate-ai-autopilot-workflows - category: Collate AI / How to Use Collate AI url: /collateai/how-to-use-collate-ai - category: Collate AI / Collate AI Tier Agent