mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2026-02-04 09:52:55 +00:00
743ff85ee4
* init pipeline-profiler cli command * Fixed issues with the bulk sink * Update generated TypeScript types * fix and remove unnecessary code blocks * fix and remove unnecessary code blocks * Added get observ data by pipeline id api * Added APIs for metrics and charts * remove fallback mechanism * Build fixes * mvn build fixes * Api remove unnecssary changes * Fix Metrics API * Fix trends API * Fixed filtering * Added sample data * Added more sample data * Move to metadata workflow * removed unused files * remove unnecesary files * json2ts * change to debug logs * remove pipeline profiler helpers * Update generated TypeScript types * Update generated TypeScript types * created PipelineExecutionIndex * Fix limit param for pagination * Update generated TypeScript types * addressed comments * linting * fix sample_data * Added serviceType in api response * Add endtime in sample data * Update generated TypeScript types * Addressed comments --------- Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Harshit Shah <harshit.shah@getcollate.io>
241 lines
7.4 KiB
JSON
241 lines
7.4 KiB
JSON
[
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.raw_customer",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.dim_address",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select * from ecommerce_db.shopify.raw_customer",
|
|
"edge_meta": { "fqn": "sample_airflow.dim_address_etl", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.raw_order",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.\"dim.product\"",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select * from sample_data.ecommerce_db.shopify.raw_order",
|
|
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.raw_order",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.\"dim.product.variant\"",
|
|
"type": "table"
|
|
},
|
|
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" },
|
|
"sql_query": "select * from sample_data.ecommerce_db.shopify.raw_order"
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.raw_customer",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.\"dim.product\"",
|
|
"type": "table"
|
|
},
|
|
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" },
|
|
"sql_query": "insert into sample_data.ecommerce_db.shopify.\"dim.product\" as select * from sample_data.ecommerce_db.shopify.raw_customer"
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.raw_customer",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.\"dim.product.variant\"",
|
|
"type": "table"
|
|
},
|
|
"edge_meta": { "fqn": "sample_airflow.dim_product_etl", "type": "pipeline" },
|
|
"sql_query": "create ecommerce_db.shopify.\"dim.product.variant\" as select * from sample_data.ecommerce_db.shopify.raw_customer"
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_superset.model.103",
|
|
"type": "dashboardDataModel"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_superset.9",
|
|
"type": "dashboard"
|
|
},
|
|
"edge_meta": { "fqn": "", "type": "" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_superset.model.73",
|
|
"type": "dashboardDataModel"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_superset.9",
|
|
"type": "dashboard"
|
|
},
|
|
"edge_meta": { "fqn": "", "type": "" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.raw_customer",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "mysql_sample.default.posts_db.Tags",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select * from mysql_sample.default.posts_db.Tags",
|
|
"edge_meta": { "fqn": "sample_airflow.dim_address_etl", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.raw_customer",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "mysql_sample.default.posts_db.Tags",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select * from mysql_sample.default.posts_db.Tags",
|
|
"edge_meta": { "fqn": "sample_airflow.dim_address_etl", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.dim_address",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "mysql_sample.default.posts_db.Tags",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select * from mysql_sample.default.posts_db.Tags",
|
|
"edge_meta": { "fqn": "sample_airflow.dim_address_etl", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.raw_customer",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.dim_customer",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select customer_id, first_name, last_name, email from raw_customer",
|
|
"edge_meta": { "fqn": "sample_dbtcloud.dbt_staging_shopify", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.raw_order",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.dim_address",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select * from raw_order where order_status = 'completed'",
|
|
"edge_meta": { "fqn": "sample_dbtcloud.dbt_staging_shopify", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.\"dim.product\"",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.fact_orders",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select product_id, order_id, quantity, price from dim.product",
|
|
"edge_meta": { "fqn": "sample_dbtcloud.dbt_transform_orders", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.dim_customer",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.fact_orders",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select customer_id, order_date, total_amount from dim_customer join orders",
|
|
"edge_meta": { "fqn": "sample_dbtcloud.dbt_transform_orders", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.\"dim.product\"",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.fact_orders",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select product_id, order_id, quantity, price, discount from dim.product",
|
|
"edge_meta": { "fqn": "sample_dbtcloud.dbt_transform_orders", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.dim_customer",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.fact_orders",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select customer_id, order_date, total_amount, product_count from dim_customer",
|
|
"edge_meta": { "fqn": "sample_dbtcloud.dbt_transform_orders", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.fact_orders",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.dim_customer",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select customer_id, sum(total_amount) as lifetime_value from fact_orders group by customer_id",
|
|
"edge_meta": { "fqn": "sample_dbtcloud.dbt_analytics_customers", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.fact_orders",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.customer_features",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select customer_id, count(*) as total_orders, avg(total_amount) as avg_order_value from fact_orders group by customer_id",
|
|
"edge_meta": { "fqn": "sample_dbtcloud.dbt_ml_features", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.dim_customer",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.customer_features",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select customer_id, preferred_category, churn_probability from dim_customer",
|
|
"edge_meta": { "fqn": "sample_dbtcloud.dbt_ml_features", "type": "pipeline" }
|
|
},
|
|
{
|
|
"from": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.fact_orders",
|
|
"type": "table"
|
|
},
|
|
"to": {
|
|
"fqn": "sample_data.ecommerce_db.shopify.customer_features",
|
|
"type": "table"
|
|
},
|
|
"sql_query": "select customer_id, sum(total_amount) as total_spend from fact_orders group by customer_id",
|
|
"edge_meta": { "fqn": "sample_airflow.ml_feature_pipeline", "type": "pipeline" }
|
|
}
|
|
]
|