mirror of
https://github.com/open-metadata/OpenMetadata.git
synced 2026-01-09 05:56:17 +00:00
743ff85ee4
* init pipeline-profiler cli command * Fixed issues with the bulk sink * Update generated TypeScript types * fix and remove unnecessary code blocks * fix and remove unnecessary code blocks * Added get observ data by pipeline id api * Added APIs for metrics and charts * remove fallback mechanism * Build fixes * mvn build fixes * Api remove unnecssary changes * Fix Metrics API * Fix trends API * Fixed filtering * Added sample data * Added more sample data * Move to metadata workflow * removed unused files * remove unnecesary files * json2ts * change to debug logs * remove pipeline profiler helpers * Update generated TypeScript types * Update generated TypeScript types * created PipelineExecutionIndex * Fix limit param for pagination * Update generated TypeScript types * addressed comments * linting * fix sample_data * Added serviceType in api response * Add endtime in sample data * Update generated TypeScript types * Addressed comments --------- Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Harshit Shah <harshit.shah@getcollate.io>
239 lines
7.0 KiB
JSON
239 lines
7.0 KiB
JSON
{
|
|
"tables": [
|
|
{
|
|
"tableFqn": "sample_data.ecommerce_db.shopify.dim_address",
|
|
"pipelineObservability": [
|
|
{
|
|
"pipeline": "sample_airflow.dim_address_etl",
|
|
"scheduleInterval": "0 10 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763402163120,
|
|
"lastRunStatus": "Successful"
|
|
},
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_staging_shopify",
|
|
"scheduleInterval": "0 1 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763402163120,
|
|
"lastRunStatus": "Successful"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"tableFqn": "sample_data.ecommerce_db.shopify.dim_product",
|
|
"pipelineObservability": [
|
|
{
|
|
"pipeline": "sample_airflow.dim_product_etl",
|
|
"scheduleInterval": "0 10 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763636163120,
|
|
"lastRunStatus": "Successful"
|
|
},
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_transform_orders",
|
|
"scheduleInterval": "0 3 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763636163120,
|
|
"lastRunStatus": "Successful"
|
|
},
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_snapshot_inventory",
|
|
"scheduleInterval": "0 * * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763733363120,
|
|
"lastRunStatus": "Successful"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"tableFqn": "sample_data.ecommerce_db.shopify.dim_customer",
|
|
"pipelineObservability": [
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_staging_shopify",
|
|
"scheduleInterval": "0 1 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763402163120,
|
|
"lastRunStatus": "Successful"
|
|
},
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_transform_orders",
|
|
"scheduleInterval": "0 3 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763636163120,
|
|
"lastRunStatus": "Successful"
|
|
},
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_analytics_customers",
|
|
"scheduleInterval": "0 4 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763640163120,
|
|
"lastRunStatus": "Successful"
|
|
},
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_ml_features",
|
|
"scheduleInterval": "0 2,14 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763711763120,
|
|
"lastRunStatus": "Successful"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"tableFqn": "sample_data.ecommerce_db.shopify.fact_orders",
|
|
"pipelineObservability": [
|
|
{
|
|
"pipeline": "sample_airflow.presto_etl",
|
|
"scheduleInterval": "0 */4 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763729763120,
|
|
"lastRunStatus": "Failed"
|
|
},
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_transform_orders",
|
|
"scheduleInterval": "0 3 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763636163120,
|
|
"lastRunStatus": "Successful"
|
|
},
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_test_data_quality",
|
|
"scheduleInterval": "0 */6 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763720163120,
|
|
"lastRunStatus": "Successful"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"tableFqn": "sample_data.ecommerce_db.shopify.fact_orders",
|
|
"pipelineObservability": [
|
|
{
|
|
"pipeline": "sample_airflow.snowflake_etl",
|
|
"scheduleInterval": "0 */6 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763420163120,
|
|
"lastRunStatus": "Successful"
|
|
},
|
|
{
|
|
"pipeline": "sample_airflow.ml_feature_pipeline",
|
|
"scheduleInterval": "0 2,14 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763711763120,
|
|
"lastRunStatus": "Successful"
|
|
},
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_transform_orders",
|
|
"scheduleInterval": "0 3 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763636163120,
|
|
"lastRunStatus": "Successful"
|
|
},
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_analytics_customers",
|
|
"scheduleInterval": "0 4 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763640163120,
|
|
"lastRunStatus": "Successful"
|
|
},
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_ml_features",
|
|
"scheduleInterval": "0 2,14 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763711763120,
|
|
"lastRunStatus": "Successful"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"tableFqn": "sample_data.ecommerce_db.shopify.fact_transactions",
|
|
"pipelineObservability": [
|
|
{
|
|
"pipeline": "sample_airflow.hive_etl",
|
|
"scheduleInterval": "0 3 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1762736163120,
|
|
"lastRunStatus": "Successful"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"tableFqn": "sample_data.ecommerce_db.shopify.dim_location",
|
|
"pipelineObservability": [
|
|
{
|
|
"pipeline": "sample_airflow.dim_location_etl",
|
|
"scheduleInterval": "0 10 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1762793763120,
|
|
"lastRunStatus": "Failed"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"tableFqn": "sample_data.ecommerce_db.shopify.fact_events",
|
|
"pipelineObservability": [
|
|
{
|
|
"pipeline": "sample_airflow.trino_etl",
|
|
"scheduleInterval": "0 * * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763445363120,
|
|
"lastRunStatus": "Successful"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"tableFqn": "sample_data.ecommerce_db.shopify.metrics_aggregated",
|
|
"pipelineObservability": [
|
|
{
|
|
"pipeline": "sample_airflow.real_time_metrics",
|
|
"scheduleInterval": "*/15 * * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763733363120,
|
|
"lastRunStatus": "Successful"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"tableFqn": "sample_data.ecommerce_db.shopify.customer_features",
|
|
"pipelineObservability": [
|
|
{
|
|
"pipeline": "sample_airflow.ml_feature_pipeline",
|
|
"scheduleInterval": "0 2,14 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1762592163120,
|
|
"lastRunStatus": "Successful"
|
|
},
|
|
{
|
|
"pipeline": "sample_dbtcloud.dbt_ml_features",
|
|
"scheduleInterval": "0 2,14 * * *",
|
|
"startTime": 1756050963120,
|
|
"endTime": 1764050963120,
|
|
"lastRunTime": 1763711763120,
|
|
"lastRunStatus": "Successful"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|