diff --git a/openmetadata-docs/content/connectors/database/athena/cli.md b/openmetadata-docs/content/connectors/database/athena/cli.md index eec5de331a2..b9fb268343f 100644 --- a/openmetadata-docs/content/connectors/database/athena/cli.md +++ b/openmetadata-docs/content/connectors/database/athena/cli.md @@ -395,6 +395,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/azuresql/cli.md b/openmetadata-docs/content/connectors/database/azuresql/cli.md index a0d8fef65a2..58dab00a733 100644 --- a/openmetadata-docs/content/connectors/database/azuresql/cli.md +++ b/openmetadata-docs/content/connectors/database/azuresql/cli.md @@ -388,6 +388,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/bigquery/cli.md b/openmetadata-docs/content/connectors/database/bigquery/cli.md index f6432c6ac4b..bee941d1bd5 100644 --- a/openmetadata-docs/content/connectors/database/bigquery/cli.md +++ b/openmetadata-docs/content/connectors/database/bigquery/cli.md @@ -562,6 +562,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/clickhouse/cli.md b/openmetadata-docs/content/connectors/database/clickhouse/cli.md index 600e46d7068..06f407a8a8d 100644 --- a/openmetadata-docs/content/connectors/database/clickhouse/cli.md +++ b/openmetadata-docs/content/connectors/database/clickhouse/cli.md @@ -476,6 +476,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/databricks/cli.md b/openmetadata-docs/content/connectors/database/databricks/cli.md index b24459d599c..ab85871a64d 100644 --- a/openmetadata-docs/content/connectors/database/databricks/cli.md +++ b/openmetadata-docs/content/connectors/database/databricks/cli.md @@ -468,6 +468,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/db2/cli.md b/openmetadata-docs/content/connectors/database/db2/cli.md index 84bf84f8964..056f727831b 100644 --- a/openmetadata-docs/content/connectors/database/db2/cli.md +++ b/openmetadata-docs/content/connectors/database/db2/cli.md @@ -384,6 +384,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/druid/cli.md b/openmetadata-docs/content/connectors/database/druid/cli.md index 9da4182c4c1..89daf3a4dbe 100644 --- a/openmetadata-docs/content/connectors/database/druid/cli.md +++ b/openmetadata-docs/content/connectors/database/druid/cli.md @@ -384,6 +384,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/hive/cli.md b/openmetadata-docs/content/connectors/database/hive/cli.md index 682c5d4468d..451871fa199 100644 --- a/openmetadata-docs/content/connectors/database/hive/cli.md +++ b/openmetadata-docs/content/connectors/database/hive/cli.md @@ -384,6 +384,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/mariadb/cli.md b/openmetadata-docs/content/connectors/database/mariadb/cli.md index 0cb9c104772..dc0e42ca6ff 100644 --- a/openmetadata-docs/content/connectors/database/mariadb/cli.md +++ b/openmetadata-docs/content/connectors/database/mariadb/cli.md @@ -383,6 +383,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/mssql/cli.md b/openmetadata-docs/content/connectors/database/mssql/cli.md index 022f6f929cf..6874dda1bd7 100644 --- a/openmetadata-docs/content/connectors/database/mssql/cli.md +++ b/openmetadata-docs/content/connectors/database/mssql/cli.md @@ -477,6 +477,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/mysql/cli.md b/openmetadata-docs/content/connectors/database/mysql/cli.md index 4be6d20a3df..1fcd2890099 100644 --- a/openmetadata-docs/content/connectors/database/mysql/cli.md +++ b/openmetadata-docs/content/connectors/database/mysql/cli.md @@ -383,6 +383,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/oracle/cli.md b/openmetadata-docs/content/connectors/database/oracle/cli.md index f82377be39d..4a02c20e730 100644 --- a/openmetadata-docs/content/connectors/database/oracle/cli.md +++ b/openmetadata-docs/content/connectors/database/oracle/cli.md @@ -394,6 +394,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/postgres/cli.md b/openmetadata-docs/content/connectors/database/postgres/cli.md index 6f7a364d415..75d5a9af9b6 100644 --- a/openmetadata-docs/content/connectors/database/postgres/cli.md +++ b/openmetadata-docs/content/connectors/database/postgres/cli.md @@ -480,6 +480,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/presto/cli.md b/openmetadata-docs/content/connectors/database/presto/cli.md index ce40876c26d..192e19a8d03 100644 --- a/openmetadata-docs/content/connectors/database/presto/cli.md +++ b/openmetadata-docs/content/connectors/database/presto/cli.md @@ -390,6 +390,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/redshift/cli.md b/openmetadata-docs/content/connectors/database/redshift/cli.md index dd87654ba27..5a32a52e92a 100644 --- a/openmetadata-docs/content/connectors/database/redshift/cli.md +++ b/openmetadata-docs/content/connectors/database/redshift/cli.md @@ -487,6 +487,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/singlestore/cli.md b/openmetadata-docs/content/connectors/database/singlestore/cli.md index 1e2aeadf973..f0833c6b77b 100644 --- a/openmetadata-docs/content/connectors/database/singlestore/cli.md +++ b/openmetadata-docs/content/connectors/database/singlestore/cli.md @@ -383,6 +383,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/snowflake/cli.md b/openmetadata-docs/content/connectors/database/snowflake/cli.md index dc8737651a0..6f300cd5409 100644 --- a/openmetadata-docs/content/connectors/database/snowflake/cli.md +++ b/openmetadata-docs/content/connectors/database/snowflake/cli.md @@ -506,6 +506,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/trino/cli.md b/openmetadata-docs/content/connectors/database/trino/cli.md index 69874869668..fd0c363073e 100644 --- a/openmetadata-docs/content/connectors/database/trino/cli.md +++ b/openmetadata-docs/content/connectors/database/trino/cli.md @@ -397,6 +397,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/connectors/database/vertica/cli.md b/openmetadata-docs/content/connectors/database/vertica/cli.md index e73c2489057..54ae4973d17 100644 --- a/openmetadata-docs/content/connectors/database/vertica/cli.md +++ b/openmetadata-docs/content/connectors/database/vertica/cli.md @@ -383,6 +383,11 @@ processor: # - MEAN # - MEDIAN # - ... + # partitionConfig: + # enablePartitioning: + # partitionColumnName: + # partitionInterval: + # partitionIntervalUnit: sink: type: metadata-rest config: {} diff --git a/openmetadata-docs/content/menu.md b/openmetadata-docs/content/menu.md index 1f275d6ea4e..3d8a594e05c 100644 --- a/openmetadata-docs/content/menu.md +++ b/openmetadata-docs/content/menu.md @@ -576,6 +576,9 @@ site_menu: - category: Features / Glossaries url: /openmetadata/glossaries + - category: Features / Data Insights + url: /openmetadata/data-insight + - category: Main Concepts url: /main-concepts color: violet-70 diff --git a/openmetadata-docs/content/openmetadata/data-insight/index.md b/openmetadata-docs/content/openmetadata/data-insight/index.md new file mode 100644 index 00000000000..cac693473b9 --- /dev/null +++ b/openmetadata-docs/content/openmetadata/data-insight/index.md @@ -0,0 +1,168 @@ +--- +title: Data Insights +slug: /openmetadata/data-insight +--- + +# Data Insights +Platform adoption is an important element for teams implementing OpenMetadata. With the data insights feature organization can drive the adoption of OpenMetadata by monitoring its usage and setting up company wide KPIs. + +## Data Insight Reports +OpenMetadata offers a suite of reports providing platform analytics around specific areas. + +### Data Assets +The Data Assets reports display important metrics around your data assets in OpenMetadata. + +**Total Data Assets** +This chart represents the total number of data assets present in OpenMetadata. It offers a view of your data assets broken down by asset type (i.e. Database, Table, ML Model, etc.) + +Total Data Assets Chart + +**Percentage of Data Assets with Description** +This chart represents the percentage of data assets present in OpenMetadata with a description. For Table asset type, this condition is true only if the table and column description are filed. It allows you to quickly view the description coverage for your data assets in OpenMetadata. + +Percentage of Assets with Description + +**Percentage of Data Assets with Owners** +This chart represents the percentage of data assets present in OpenMetadata with an owner assigned. Data assets that do not support assigning an owner will not be counted in this percentage. It allows you to quickly view the ownership coverage for your data assets in OpenMetadata. + +Percentage of Assets with Owner Assigned + +**Total Data Assets by Tier** +This chart represents a broken down view of data assets by Tiers. Data Assets with no tiers assigned are not included in this. It allows you to quickly view the breakdown of data assets by tier. + +Data Asset by Tier + +### App Analytics +The App Analytics report provides important metrics around the usage of OpenMetadata. + +**Most Viewed Data Assets** +This chart shows the top 10 data assets the most viewed in your platform. It offers a quick view to understand what are the data assets with the most interest in your organization. + +Most Viewed Assets + +**Page views by data assets** +This chart shows the total number of page views by asset type. This allows you to understand which asset familly drives the most interest in your organization + +Page Views by Assets + +**Daily active users on the platform** +This chart shows the number of daily active users on your platform. Active users are users with at least one session. This report allows to understand the platform usage and see how your organization leverage OpenMetadata. + +Daily Active Users + +**Most Active Users** +This chart shows the top 10 most active users. These users are your power users in your organization. They can be turned into evangelist to promote OpenMetadata inside your company. + +Daily Active Users + +### Setting up Data Insight Workflow +**Step 1** +Navigate to `settings > Metadata > OpenMetadata Service`. + +Metadata Service Page + +On the `OpenMetadata Service` click on `Add Ingestion > Add Data Insight Ingestion` + +Add Data Insight Ingestion + +**Step 2** +Pick a name for your ingestion workflow or leave it as is. + +Data Insight Ingestion Name + +Add any elasticsearch configuration relevant to your setup. Note that if you are deploying OpenMetadata with no custom elasticsearch deployment you can skip this configuration step. + +Data Insight Ingestion ES Config + +Choose a schedule exection time for your workflow. The schedule time is displayed in UTC. We recommend to run this workflow overnight or when activity on the platform is at its lowest to ensure accurate data. + +Data Insight Ingestion Schedule + +**Step 3** +Navigate to the `Insights` page. You should see your data insights reports. Note that if you have just deployed OpenMetadata, `App Analytic` data might not be present. `App Analytic` data are fetched from the previous day (UTC). + +## Data Insight KPIs +While data insights reports gives an analytical view of OpenMetadata platform, KPIs are here to drive platform adoption. + +Data Insight KPI + +### KPIs Categories + +**Completed Description** +Available as an absolute or relative (percentage) value, this KPI measures the description coverage of your data assets in OpenMetadata. + +**Completed Ownership** +Available as an absolute or relative (percentage) value, this KPI measures the ownershi[] coverage of your data assets in OpenMetadata. + +### Adding KPIs +On the `Insights` page, click on `Add KPI`. This will open the KPI configuration page where the following required configuration elements need to be set: +- `Name`: name of your KPI +- `Select a chart`: this links the KPI to one of the chart present in the data insight reports +- `Select a metric type`: you can choose between `PERCENTAGE` or `NUMBER`. The former will be a relative value while the latter an absolute value +- `Start date` / `End date`: this will determine the start and end date of your KPI. It sets an objective for your organization + +KPI Configuration + + + + diff --git a/openmetadata-docs/images/openmetadata/data-insight/configure-kpi.png b/openmetadata-docs/images/openmetadata/data-insight/configure-kpi.png new file mode 100644 index 00000000000..fdbc9d896af Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/configure-kpi.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/daily-active-users.png b/openmetadata-docs/images/openmetadata/data-insight/daily-active-users.png new file mode 100644 index 00000000000..fec61ca97d2 Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/daily-active-users.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/data-assets-by-tier.png b/openmetadata-docs/images/openmetadata/data-insight/data-assets-by-tier.png new file mode 100644 index 00000000000..99f5a34d4cd Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/data-assets-by-tier.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/data-insight-add-ingestion.png b/openmetadata-docs/images/openmetadata/data-insight/data-insight-add-ingestion.png new file mode 100644 index 00000000000..e85799f4c6d Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/data-insight-add-ingestion.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/data-insight-ingestion-es-config.png b/openmetadata-docs/images/openmetadata/data-insight/data-insight-ingestion-es-config.png new file mode 100644 index 00000000000..2d9e0351f4d Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/data-insight-ingestion-es-config.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/data-insight-ingestion-name.png b/openmetadata-docs/images/openmetadata/data-insight/data-insight-ingestion-name.png new file mode 100644 index 00000000000..ef3e85671e6 Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/data-insight-ingestion-name.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/data-insight-ingestion-schedule.png b/openmetadata-docs/images/openmetadata/data-insight/data-insight-ingestion-schedule.png new file mode 100644 index 00000000000..8148d20029d Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/data-insight-ingestion-schedule.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/data-insight-kpi.png b/openmetadata-docs/images/openmetadata/data-insight/data-insight-kpi.png new file mode 100644 index 00000000000..dc9722b1d14 Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/data-insight-kpi.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/metadata-nav.png b/openmetadata-docs/images/openmetadata/data-insight/metadata-nav.png new file mode 100644 index 00000000000..a26f3af6500 Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/metadata-nav.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/most-active-users.png b/openmetadata-docs/images/openmetadata/data-insight/most-active-users.png new file mode 100644 index 00000000000..7cd5c5d099b Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/most-active-users.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/most-viewed-assets.png b/openmetadata-docs/images/openmetadata/data-insight/most-viewed-assets.png new file mode 100644 index 00000000000..f46df5c64f2 Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/most-viewed-assets.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/percentage-description.png b/openmetadata-docs/images/openmetadata/data-insight/percentage-description.png new file mode 100644 index 00000000000..5f24763c8a7 Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/percentage-description.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/percentage-owner.png b/openmetadata-docs/images/openmetadata/data-insight/percentage-owner.png new file mode 100644 index 00000000000..f170a38098a Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/percentage-owner.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/total-data-assets.png b/openmetadata-docs/images/openmetadata/data-insight/total-data-assets.png new file mode 100644 index 00000000000..b6669c6972b Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/total-data-assets.png differ diff --git a/openmetadata-docs/images/openmetadata/data-insight/views-by-assets.png b/openmetadata-docs/images/openmetadata/data-insight/views-by-assets.png new file mode 100644 index 00000000000..aa43c7e7b6e Binary files /dev/null and b/openmetadata-docs/images/openmetadata/data-insight/views-by-assets.png differ