datahub/metadata-integration/java/spark-lineage-legacy/spark-smoke-test/golden_json/JavaHdfsIn2HiveCreateInsertTable.json

{
  "urn:li:dataFlow:(spark,JavaHdfsIn2HiveCreateInsertTable,spark_spark-master_7077)": {
    "value": {
      "com.linkedin.metadata.snapshot.DataFlowSnapshot": {
        "urn": "urn:li:dataFlow:(spark,JavaHdfsIn2HiveCreateInsertTable,spark_spark-master_7077)",
        "aspects": [
          {
            "com.linkedin.metadata.key.DataFlowKey": {
              "orchestrator": "spark",
              "cluster": "spark_spark-master_7077",
              "flowId": "JavaHdfsIn2HiveCreateInsertTable"
            }
          },
          {
            "com.linkedin.common.DataPlatformInstance": {
              "platform": "urn:li:dataPlatform:spark"
            }
          },
          {
            "com.linkedin.datajob.DataFlowInfo": {
              "name": "JavaHdfsIn2HiveCreateInsertTable",
              "customProperties": {
                "sparkUser": "root",
                "appName": "JavaHdfsIn2HiveCreateInsertTable"
              }
            }
          },
          {
            "com.linkedin.common.BrowsePaths": {
              "paths": [
                "/spark/spark_spark-master_7077"
              ]
            }
          }
        ]
      }
    }
  },
  "urn:li:dataJob:(urn:li:dataFlow:(spark,JavaHdfsIn2HiveCreateInsertTable,spark_spark-master_7077),QueryExecId_5)": {
    "value": {
      "com.linkedin.metadata.snapshot.DataJobSnapshot": {
        "urn": "urn:li:dataJob:(urn:li:dataFlow:(spark,JavaHdfsIn2HiveCreateInsertTable,spark_spark-master_7077),QueryExecId_5)",
        "aspects": [
          {
            "com.linkedin.metadata.key.DataJobKey": {
              "jobId": "QueryExecId_5",
              "flow": "urn:li:dataFlow:(spark,JavaHdfsIn2HiveCreateInsertTable,spark_spark-master_7077)"
            }
          },
          {
            "com.linkedin.datajob.DataJobInputOutput": {
              "inputDatasets": [
                "urn:li:dataset:(urn:li:dataPlatform:file,file:/opt/workspace/resources/data/in1.csv,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:file,file:/opt/workspace/resources/data/in2.csv,PROD)"
              ],
              "outputDatasets": [
                "urn:li:dataset:(urn:li:dataPlatform:hive,JavaHdfsIn2HiveCreateInsertTable.foo4,PROD)"
              ]
            }
          },
          {
            "com.linkedin.common.BrowsePaths": {
              "paths": [
                "/spark/spark_spark-master_7077"
              ]
            }
          },
          {
            "com.linkedin.common.DataPlatformInstance": {
              "platform": "urn:li:dataPlatform:spark"
            }
          },
          {
            "com.linkedin.datajob.DataJobInfo": {
              "name": "saveAsTable at HdfsIn2HiveCreateInsertTable.java:28",
              "type": {
                "string": "sparkJob"
              },
              "customProperties": {
                "SQLQueryId": "5",
                "appName": "JavaHdfsIn2HiveCreateInsertTable",
                "description": "saveAsTable at HdfsIn2HiveCreateInsertTable.java:28",
                "queryPlan": "CreateDataSourceTableAsSelectCommand `JavaHdfsIn2HiveCreateInsertTable`.`foo4`, Overwrite, [a, b, c, d]\n+- Project [a#16, b#20, c#40, d#44]\n   +- Join Inner, (id#10 = id#34)\n      :- Project [id#10, c1#11 AS a#16, c2#12 AS b#20]\n      :  +- Filter isnotnull(id#10)\n      :     +- Relation[id#10,c1#11,c2#12] csv\n      +- Project [id#34, c1#35 AS c#40, c2#36 AS d#44]\n         +- Filter isnotnull(id#34)\n            +- Relation[id#34,c1#35,c2#36] csv\n"
              }
            }
          }
        ]
      }
    }
  },
  "urn:li:dataJob:(urn:li:dataFlow:(spark,JavaHdfsIn2HiveCreateInsertTable,spark_spark-master_7077),QueryExecId_6)": {
    "value": {
      "com.linkedin.metadata.snapshot.DataJobSnapshot": {
        "urn": "urn:li:dataJob:(urn:li:dataFlow:(spark,JavaHdfsIn2HiveCreateInsertTable,spark_spark-master_7077),QueryExecId_6)",
        "aspects": [
          {
            "com.linkedin.metadata.key.DataJobKey": {
              "jobId": "QueryExecId_6",
              "flow": "urn:li:dataFlow:(spark,JavaHdfsIn2HiveCreateInsertTable,spark_spark-master_7077)"
            }
          },
          {
            "com.linkedin.common.DataPlatformInstance": {
              "platform": "urn:li:dataPlatform:spark"
            }
          },
          {
            "com.linkedin.common.BrowsePaths": {
              "paths": [
                "/spark/spark_spark-master_7077"
              ]
            }
          },
          {
            "com.linkedin.datajob.DataJobInputOutput": {
              "inputDatasets": [
                "urn:li:dataset:(urn:li:dataPlatform:file,file:/opt/workspace/resources/data/in1.csv,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:file,file:/opt/workspace/resources/data/in2.csv,PROD)"
              ],
              "outputDatasets": [
                "urn:li:dataset:(urn:li:dataPlatform:hive,javahdfsin2hivecreateinserttable.foo4,PROD)"
              ]
            }
          },
          {
            "com.linkedin.datajob.DataJobInfo": {
              "name": "saveAsTable at HdfsIn2HiveCreateInsertTable.java:29",
              "type": {
                "string": "sparkJob"
              },
              "customProperties": {
                "SQLQueryId": "6",
                "appName": "JavaHdfsIn2HiveCreateInsertTable",
                "description": "saveAsTable at HdfsIn2HiveCreateInsertTable.java:29",
                "queryPlan": "CreateDataSourceTableAsSelectCommand `javahdfsin2hivecreateinserttable`.`foo4`, org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe, Append, [a, b, c, d]\n+- Project [a#16, b#20, c#40, d#44]\n   +- Join Inner, (id#10 = id#34)\n      :- Project [id#10, c1#11 AS a#16, c2#12 AS b#20]\n      :  +- Filter isnotnull(id#10)\n      :     +- Relation[id#10,c1#11,c2#12] csv\n      +- Project [id#34, c1#35 AS c#40, c2#36 AS d#44]\n         +- Filter isnotnull(id#34)\n            +- Relation[id#34,c1#35,c2#36] csv\n"
              }
            }
          }
        ]
      }
    }
  },
  "urn:li:dataJob:(urn:li:dataFlow:(spark,JavaHdfsIn2HiveCreateInsertTable,spark_spark-master_7077),QueryExecId_7)": {
    "value": {
      "com.linkedin.metadata.snapshot.DataJobSnapshot": {
        "urn": "urn:li:dataJob:(urn:li:dataFlow:(spark,JavaHdfsIn2HiveCreateInsertTable,spark_spark-master_7077),QueryExecId_7)",
        "aspects": [
          {
            "com.linkedin.metadata.key.DataJobKey": {
              "jobId": "QueryExecId_7",
              "flow": "urn:li:dataFlow:(spark,JavaHdfsIn2HiveCreateInsertTable,spark_spark-master_7077)"
            }
          },
          {
            "com.linkedin.datajob.DataJobInfo": {
              "name": "insertInto at HdfsIn2HiveCreateInsertTable.java:30",
              "type": {
                "string": "sparkJob"
              },
              "customProperties": {
                "SQLQueryId": "7",
                "appName": "JavaHdfsIn2HiveCreateInsertTable",
                "description": "insertInto at HdfsIn2HiveCreateInsertTable.java:30"
              }
            }
          },
          {
            "com.linkedin.common.BrowsePaths": {
              "paths": [
                "/spark/spark_spark-master_7077"
              ]
            }
          },
          {
            "com.linkedin.common.DataPlatformInstance": {
              "platform": "urn:li:dataPlatform:spark"
            }
          },
          {
            "com.linkedin.datajob.DataJobInputOutput": {
              "inputDatasets": [
                "urn:li:dataset:(urn:li:dataPlatform:file,file:/opt/workspace/resources/data/in1.csv,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:file,file:/opt/workspace/resources/data/in2.csv,PROD)"
              ],
              "outputDatasets": [
                "urn:li:dataset:(urn:li:dataPlatform:hive,javahdfsin2hivecreateinserttable.foo4,PROD)"
              ]
            }
          }
        ]
      }
    }
  }
}