datahub/metadata-integration/java/spark-lineage-legacy/spark-smoke-test/golden_json/PythonHdfsIn2HdfsOut1.json

{
  "urn:li:dataFlow:(spark,PythonHdfsIn2HdfsOut1,spark_spark-master_7077)": {
    "value": {
      "com.linkedin.metadata.snapshot.DataFlowSnapshot": {
        "urn": "urn:li:dataFlow:(spark,PythonHdfsIn2HdfsOut1,spark_spark-master_7077)",
        "aspects": [
          {
            "com.linkedin.metadata.key.DataFlowKey": {
              "orchestrator": "spark",
              "cluster": "spark_spark-master_7077",
              "flowId": "PythonHdfsIn2HdfsOut1"
            }
          },
          {
            "com.linkedin.common.DataPlatformInstance": {
              "platform": "urn:li:dataPlatform:spark"
            }
          },
          {
            "com.linkedin.datajob.DataFlowInfo": {
              "name": "PythonHdfsIn2HdfsOut1",
              "customProperties": {
                "sparkUser": "root",
                "appName": "PythonHdfsIn2HdfsOut1"
              }
            }
          },
          {
            "com.linkedin.common.BrowsePaths": {
              "paths": [
                "/spark/spark_spark-master_7077"
              ]
            }
          }
        ]
      }
    }
  },
  "urn:li:dataJob:(urn:li:dataFlow:(spark,PythonHdfsIn2HdfsOut1,spark_spark-master_7077),QueryExecId_4)": {
    "value": {
      "com.linkedin.metadata.snapshot.DataJobSnapshot": {
        "urn": "urn:li:dataJob:(urn:li:dataFlow:(spark,PythonHdfsIn2HdfsOut1,spark_spark-master_7077),QueryExecId_4)",
        "aspects": [
          {
            "com.linkedin.metadata.key.DataJobKey": {
              "jobId": "QueryExecId_4",
              "flow": "urn:li:dataFlow:(spark,PythonHdfsIn2HdfsOut1,spark_spark-master_7077)"
            }
          },
          {
            "com.linkedin.datajob.DataJobInputOutput": {
              "inputDatasets": [
                "urn:li:dataset:(urn:li:dataPlatform:file,file:/opt/workspace/resources/data/in1.csv,PROD)",
                "urn:li:dataset:(urn:li:dataPlatform:file,file:/opt/workspace/resources/data/in2.csv,PROD)"
              ],
              "outputDatasets": [
                "urn:li:dataset:(urn:li:dataPlatform:file,file:/opt/workspace/resources/data/PythonHdfsIn2HdfsOut1/out.csv,PROD)"
              ]
            }
          },
          {
            "com.linkedin.common.BrowsePaths": {
              "paths": [
                "/spark/spark_spark-master_7077"
              ]
            }
          },
          {
            "com.linkedin.datajob.DataJobInfo": {
              "name": "csv at NativeMethodAccessorImpl.java:0",
              "type": {
                "string": "sparkJob"
              },
              "customProperties": {
                "SQLQueryId": "4",
                "appName": "PythonHdfsIn2HdfsOut1",
                "description": "csv at NativeMethodAccessorImpl.java:0",
                "queryPlan": "InsertIntoHadoopFsRelationCommand file:/opt/workspace/resources/data/PythonHdfsIn2HdfsOut1/out.csv, false, CSV, Map(path -> ../resources/data/PythonHdfsIn2HdfsOut1/out.csv), Overwrite, [a, b, c, d]\n+- Project [c1#11 AS a#32, c2#12 AS b#33, c1#27 AS c#34, c2#28 AS d#35]\n   +- Join Inner, (id#10 = id#26)\n      :- Filter isnotnull(id#10)\n      :  +- Relation[id#10,c1#11,c2#12] csv\n      +- Filter isnotnull(id#26)\n         +- Relation[id#26,c1#27,c2#28] csv\n"
              }
            }
          },
          {
            "com.linkedin.common.DataPlatformInstance": {
              "platform": "urn:li:dataPlatform:spark"
            }
          }
        ]
      }
    }
  }
}