From 1e11851a90d055b98d22215a4be0ff7ad2489ddf Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Wed, 14 Apr 2021 19:27:01 -0700 Subject: [PATCH] feat(sample): adding sample mces for dataflows and datajobs (#2396) --- .../examples/mce_files/bootstrap_mce.json | 124 +++++++++++++++++- 1 file changed, 123 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/examples/mce_files/bootstrap_mce.json b/metadata-ingestion/examples/mce_files/bootstrap_mce.json index e4b0396876..9af1d79634 100644 --- a/metadata-ingestion/examples/mce_files/bootstrap_mce.json +++ b/metadata-ingestion/examples/mce_files/bootstrap_mce.json @@ -887,7 +887,129 @@ }, "proposedDelta": null }, - { + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "name": "User Creations", + "description": "Constructs the fct_users_created from logging_events", + "type": "SQL" + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_456)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "name": "User Deletions", + "description": "Constructs the fct_users_deleted from logging_events", + "type": "SQL" + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + ] + } + } + ] + } + }, + "proposedDelta": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(airflow,dag_abc,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "name": "Users", + "description": "Constructs the fct_users_deleted and fct_users_created tables", + "project": null + } + } + ] + } + }, + "proposedDelta": null + }, + { "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": {