Minor: Add pipeline display name in lineage export (#20004)

* Minor: Add pipeline display name in lineage export

* add to csvDocumentation as well

---------

Co-authored-by: mohitdeuex <mohit.y@deuexsolutions.com>
This commit is contained in:
Sriharsha Chintalapani 2025-02-27 00:46:44 -08:00 committed by GitHub
parent 73876a9122
commit dc0437119e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 16 additions and 1 deletions

View File

@ -307,6 +307,7 @@ public class LineageRepository {
"fromChildEntityFQN", "fromChildEntityFQN",
"toChildEntityFQN", "toChildEntityFQN",
"pipelineName", "pipelineName",
"pipelineDisplayName",
"pipelineType", "pipelineType",
"pipelineDescription", "pipelineDescription",
"pipelineOwners", "pipelineOwners",
@ -355,6 +356,7 @@ public class LineageRepository {
"", "",
"", "",
"", "",
"",
""); "");
LOG.debug( LOG.debug(
"Exported ColumnMapping: from='{}', to='{}'", "Exported ColumnMapping: from='{}', to='{}'",
@ -364,7 +366,7 @@ public class LineageRepository {
} else if (!pipeline.isMissingNode() && !pipeline.isNull()) { } else if (!pipeline.isMissingNode() && !pipeline.isNull()) {
writePipelineRow(csvWriter, baseRow, pipeline); writePipelineRow(csvWriter, baseRow, pipeline);
} else { } else {
writeCsvRow(csvWriter, baseRow, "", "", "", "", "", "", "", "", ""); writeCsvRow(csvWriter, baseRow, "", "", "", "", "", "", "", "", "", "");
} }
} }
csvWriter.close(); csvWriter.close();
@ -377,6 +379,7 @@ public class LineageRepository {
private void writePipelineRow( private void writePipelineRow(
CSVWriter csvWriter, Map<String, String> baseRow, JsonNode pipeline) { CSVWriter csvWriter, Map<String, String> baseRow, JsonNode pipeline) {
String pipelineName = getText(pipeline, "name"); String pipelineName = getText(pipeline, "name");
String pipelineDisplayName = getText(pipeline, "displayName");
String pipelineType = getText(pipeline, "serviceType"); String pipelineType = getText(pipeline, "serviceType");
String pipelineDescription = getText(pipeline, "description"); String pipelineDescription = getText(pipeline, "description");
String pipelineOwners = getOwners(pipeline.path("owners")); String pipelineOwners = getOwners(pipeline.path("owners"));
@ -390,6 +393,7 @@ public class LineageRepository {
"", "",
"", "",
pipelineName, pipelineName,
pipelineDisplayName,
pipelineType, pipelineType,
pipelineDescription, pipelineDescription,
pipelineOwners, pipelineOwners,
@ -405,6 +409,7 @@ public class LineageRepository {
String fromChildFQN, String fromChildFQN,
String toChildFQN, String toChildFQN,
String pipelineName, String pipelineName,
String pipelineDisplayName,
String pipelineType, String pipelineType,
String pipelineDescription, String pipelineDescription,
String pipelineOwners, String pipelineOwners,
@ -425,6 +430,7 @@ public class LineageRepository {
fromChildFQN, fromChildFQN,
toChildFQN, toChildFQN,
pipelineName, pipelineName,
pipelineDisplayName,
pipelineType, pipelineType,
pipelineDescription, pipelineDescription,
pipelineOwners, pipelineOwners,
@ -531,6 +537,7 @@ public class LineageRepository {
addField(recordList, getStringOrNull(edge, "description")); addField(recordList, getStringOrNull(edge, "description"));
addField(recordList, getStringOrNull(edge, "pipeline", "id")); addField(recordList, getStringOrNull(edge, "pipeline", "id"));
addField(recordList, getStringOrNull(edge, "pipeline", "fullyQualifiedName")); addField(recordList, getStringOrNull(edge, "pipeline", "fullyQualifiedName"));
addField(recordList, getStringOrNull(edge, "pipeline", "displayName"));
addField(recordList, processColumnLineage(edge)); addField(recordList, processColumnLineage(edge));
addField(recordList, getStringOrNull(edge, "sqlQuery")); addField(recordList, getStringOrNull(edge, "sqlQuery"));
addField(recordList, getStringOrNull(edge, "source")); addField(recordList, getStringOrNull(edge, "source"));

View File

@ -73,6 +73,14 @@
"`Airflow.transformation_pipeline`" "`Airflow.transformation_pipeline`"
] ]
}, },
{
"name": "pipelineDisplayName",
"required": false,
"description": "Diplaay Name of the pipeline involved in lineage",
"examples": [
"`Airflow.Test`"
]
},
{ {
"name": "columnLineage", "name": "columnLineage",
"required": false, "required": false,