mirror of
https://github.com/datahub-project/datahub.git
synced 2026-01-04 13:53:36 +00:00
refactor: remove unused PDL files (#3659)
This commit is contained in:
Dexter Lee
GitHub
parent
d5eddf52ab
commit
8757543be8
@ -3,8 +3,6 @@ package com.linkedin.mxe;
|
||||
import com.linkedin.pegasus2avro.mxe.FailedMetadataChangeEvent;
|
||||
import com.linkedin.pegasus2avro.mxe.MetadataAuditEvent;
|
||||
import com.linkedin.pegasus2avro.mxe.MetadataChangeEvent;
|
||||
import com.linkedin.pegasus2avro.mxe.MetadataGraphEvent;
|
||||
import com.linkedin.pegasus2avro.mxe.MetadataSearchEvent;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
@ -26,8 +24,6 @@ public class Configs {
|
||||
put(Topics.METADATA_AUDIT_EVENT, MetadataAuditEvent.SCHEMA$);
|
||||
put(Topics.METADATA_CHANGE_EVENT, MetadataChangeEvent.SCHEMA$);
|
||||
put(Topics.FAILED_METADATA_CHANGE_EVENT, FailedMetadataChangeEvent.SCHEMA$);
|
||||
put(Topics.METADATA_GRAPH_EVENT, MetadataGraphEvent.SCHEMA$);
|
||||
put(Topics.METADATA_SEARCH_EVENT, MetadataSearchEvent.SCHEMA$);
|
||||
|
||||
put(Topics.DEV_METADATA_AUDIT_EVENT, MetadataAuditEvent.SCHEMA$);
|
||||
put(Topics.DEV_METADATA_CHANGE_EVENT, MetadataChangeEvent.SCHEMA$);
|
||||
|
||||
@ -1,12 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
/**
|
||||
* Common fields that apply to all entities
|
||||
*/
|
||||
record BaseEntity {
|
||||
|
||||
/**
|
||||
* Whether the entity has been removed or not
|
||||
*/
|
||||
removed: optional boolean = false
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
import com.linkedin.common.ChartUrn
|
||||
|
||||
/**
|
||||
* Data model for a Chart entity
|
||||
*/
|
||||
record ChartEntity includes BaseEntity {
|
||||
|
||||
/**
|
||||
* Urn for the chart
|
||||
*/
|
||||
urn: ChartUrn
|
||||
|
||||
/**
|
||||
* Dashboard tool
|
||||
*/
|
||||
dashboardTool: optional string
|
||||
|
||||
/**
|
||||
* Chart Id
|
||||
*/
|
||||
chartId: optional string
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
import com.linkedin.common.CorpGroupUrn
|
||||
|
||||
/**
|
||||
* Data model for a CorpGroup entity(go/groupId)
|
||||
*/
|
||||
record CorpGroupEntity includes BaseEntity {
|
||||
|
||||
/**
|
||||
* Urn for the LDAP Group
|
||||
*/
|
||||
urn: CorpGroupUrn
|
||||
|
||||
/**
|
||||
* name of the group, e.g. wherehows-dev, ask_metadata
|
||||
*/
|
||||
name: optional string
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
import com.linkedin.common.CorpuserUrn
|
||||
|
||||
/**
|
||||
* Data model for a CorpUser entity
|
||||
*/
|
||||
record CorpUserEntity includes BaseEntity {
|
||||
|
||||
/**
|
||||
* Urn for the LDAP User
|
||||
*/
|
||||
urn: CorpuserUrn
|
||||
|
||||
/**
|
||||
* LDAP name(id) : e.g. hzhang2, ywang5 ..
|
||||
*/
|
||||
name: optional string
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
import com.linkedin.common.DashboardUrn
|
||||
|
||||
/**
|
||||
* Data model for a Dashboard entity
|
||||
*/
|
||||
record DashboardEntity includes BaseEntity {
|
||||
|
||||
/**
|
||||
* Urn for the dashboard
|
||||
*/
|
||||
urn: DashboardUrn
|
||||
|
||||
/**
|
||||
* Dashboard tool
|
||||
*/
|
||||
dashboardTool: optional string
|
||||
|
||||
/**
|
||||
* Dashboard Id
|
||||
*/
|
||||
dashboardId: optional string
|
||||
}
|
||||
@ -1,30 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
import com.linkedin.common.DataFlowUrn
|
||||
|
||||
/**
|
||||
* Data model for a DataFlow entity
|
||||
*/
|
||||
record DataFlowEntity includes BaseEntity {
|
||||
|
||||
/**
|
||||
* Urn for the DataFlow
|
||||
*/
|
||||
urn: DataFlowUrn
|
||||
|
||||
/**
|
||||
* Workflow orchestrator ex: Azkaban, Airflow
|
||||
*/
|
||||
orchestrator: optional string
|
||||
|
||||
/**
|
||||
* Id of the flow
|
||||
*/
|
||||
flowId: optional string
|
||||
|
||||
/**
|
||||
* Cluster of the flow
|
||||
*/
|
||||
cluster: optional string
|
||||
|
||||
}
|
||||
@ -1,27 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
import com.linkedin.common.DataJobUrn
|
||||
import com.linkedin.common.DataFlowUrn
|
||||
|
||||
|
||||
/**
|
||||
* Data model for a DataJob entity
|
||||
*/
|
||||
record DataJobEntity includes BaseEntity {
|
||||
|
||||
/**
|
||||
* Urn for the DataJob
|
||||
*/
|
||||
urn: DataJobUrn
|
||||
|
||||
/**
|
||||
* Urn of the associated DataFlow
|
||||
*/
|
||||
flow: optional DataFlowUrn
|
||||
|
||||
/**
|
||||
* Id of the job
|
||||
*/
|
||||
jobId: optional string
|
||||
|
||||
}
|
||||
@ -1,31 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
import com.linkedin.common.DataPlatformUrn
|
||||
import com.linkedin.common.FabricType
|
||||
import com.linkedin.common.DataProcessUrn
|
||||
|
||||
/**
|
||||
* Data model for a Data Process entity
|
||||
*/
|
||||
record DataProcessEntity {
|
||||
|
||||
/**
|
||||
* Urn for the Data Process
|
||||
*/
|
||||
urn: DataProcessUrn
|
||||
|
||||
/**
|
||||
* Data Process name(id)
|
||||
*/
|
||||
name: optional string
|
||||
|
||||
/**
|
||||
* Process Orchestrator for this process in the form. Options can be Airflow, Azkaban, Azure Data Factory
|
||||
*/
|
||||
orchestrator: optional string
|
||||
|
||||
/**
|
||||
* Fabric type where dataset belongs to or where it was generated.
|
||||
*/
|
||||
origin: optional FabricType
|
||||
}
|
||||
@ -1,31 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
import com.linkedin.common.DataPlatformUrn
|
||||
import com.linkedin.common.DatasetUrn
|
||||
import com.linkedin.common.FabricType
|
||||
|
||||
/**
|
||||
* Data model for a dataset entity
|
||||
*/
|
||||
record DatasetEntity includes BaseEntity {
|
||||
|
||||
/**
|
||||
* Urn for the dataset
|
||||
*/
|
||||
urn: DatasetUrn
|
||||
|
||||
/**
|
||||
* Dataset native name e.g. {db}.{table}, /dir/subdir/{name}, or {name}
|
||||
*/
|
||||
name: optional string
|
||||
|
||||
/**
|
||||
* Platform urn for the dataset in the form of urn:li:platform:{platform_name}
|
||||
*/
|
||||
platform: optional DataPlatformUrn
|
||||
|
||||
/**
|
||||
* Fabric type where dataset belongs to or where it was generated.
|
||||
*/
|
||||
origin: optional FabricType
|
||||
}
|
||||
@ -1,6 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
/**
|
||||
* A union of all supported entity types.
|
||||
*/
|
||||
typeref Entity = union[CorpUserEntity, DatasetEntity, DataProcessEntity, MLModelEntity, DataFlowEntity, DataJobEntity]
|
||||
@ -1,19 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
import com.linkedin.common.GlossaryNodeUrn
|
||||
|
||||
/**
|
||||
* Data model for a GlossaryNode entity
|
||||
*/
|
||||
record GlossaryNodeEntity includes BaseEntity {
|
||||
|
||||
/**
|
||||
* Urn for the GlossaryNode
|
||||
*/
|
||||
urn: GlossaryNodeUrn
|
||||
|
||||
/**
|
||||
* Business node name
|
||||
*/
|
||||
name: optional string
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
import com.linkedin.common.GlossaryTermUrn
|
||||
|
||||
/**
|
||||
* Data model for a GlossaryTerm entity
|
||||
*/
|
||||
record GlossaryTermEntity includes BaseEntity {
|
||||
|
||||
/**
|
||||
* Urn for the GlossaryTerm
|
||||
*/
|
||||
urn: GlossaryTermUrn
|
||||
|
||||
/**
|
||||
* Business term name
|
||||
*/
|
||||
name: optional string
|
||||
}
|
||||
@ -1,31 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
import com.linkedin.common.DataPlatformUrn
|
||||
import com.linkedin.common.FabricType
|
||||
import com.linkedin.common.MLModelUrn
|
||||
|
||||
/**
|
||||
* Data model for a ML Model entity
|
||||
*/
|
||||
record MLModelEntity includes BaseEntity {
|
||||
|
||||
/**
|
||||
* Urn for the ML Model
|
||||
*/
|
||||
urn: MLModelUrn
|
||||
|
||||
/**
|
||||
* ML Model native name
|
||||
*/
|
||||
name: optional string
|
||||
|
||||
/**
|
||||
* Platform urn for the ML Model in the form of urn:li:platform:{platform_name}
|
||||
*/
|
||||
platform: optional DataPlatformUrn
|
||||
|
||||
/**
|
||||
* Fabric type where ML Model belongs to or where it was generated.
|
||||
*/
|
||||
origin: optional FabricType
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
namespace com.linkedin.metadata.entity
|
||||
|
||||
import com.linkedin.common.TagUrn
|
||||
|
||||
/**
|
||||
* Data model for a tag entity
|
||||
*/
|
||||
record TagEntity includes BaseEntity {
|
||||
|
||||
/**
|
||||
* Urn for the tag
|
||||
*/
|
||||
urn: TagUrn
|
||||
|
||||
/**
|
||||
* Name of the tag
|
||||
*/
|
||||
name: optional string
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
namespace com.linkedin.metadata.relationship
|
||||
|
||||
import com.linkedin.common.Urn
|
||||
|
||||
/**
|
||||
* Common fields that apply to all relationships
|
||||
*/
|
||||
record BaseRelationship {
|
||||
|
||||
/**
|
||||
* Urn for the source of the relationship
|
||||
*/
|
||||
source: Urn
|
||||
|
||||
/**
|
||||
* Urn for the destination of the relationship
|
||||
*/
|
||||
destination: Urn
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
namespace com.linkedin.metadata.relationship
|
||||
|
||||
/**
|
||||
* A generic model for the
|
||||
*/
|
||||
@pairings = [ {
|
||||
"destination" : "com.linkedin.common.urn.DatasetUrn",
|
||||
"source" : "com.linkedin.common.urn.DataJobUrn"
|
||||
} ]
|
||||
record Consumes includes BaseRelationship {
|
||||
}
|
||||
@ -1,17 +0,0 @@
|
||||
namespace com.linkedin.metadata.relationship
|
||||
|
||||
/**
|
||||
* A generic model for the Has-A relationship
|
||||
*/
|
||||
@pairings = [ {
|
||||
"destination" : "com.linkedin.common.urn.ChartUrn",
|
||||
"source" : "com.linkedin.common.urn.DashboardUrn"
|
||||
},{
|
||||
"destination" : "com.linkedin.common.urn.GlossaryTermUrn",
|
||||
"source" : "com.linkedin.common.urn.GlossaryNodeUrn"
|
||||
},{
|
||||
"destination" : "com.linkedin.common.urn.GlossaryNodeUrn",
|
||||
"source" : "com.linkedin.common.urn.GlossaryNodeUrn"
|
||||
} ]
|
||||
record Contains includes BaseRelationship {
|
||||
}
|
||||
@ -1,21 +0,0 @@
|
||||
namespace com.linkedin.metadata.relationship
|
||||
|
||||
import com.linkedin.dataset.DatasetLineageType
|
||||
|
||||
/**
|
||||
* A generic model for the DownstreamOf relationship
|
||||
*/
|
||||
@pairings = [ {
|
||||
"destination" : "com.linkedin.common.urn.DatasetUrn",
|
||||
"source" : "com.linkedin.common.urn.DatasetUrn"
|
||||
}, {
|
||||
"destination" : "com.linkedin.common.urn.DatasetUrn",
|
||||
"source" : "com.linkedin.common.urn.ChartUrn"
|
||||
} ]
|
||||
record DownstreamOf includes BaseRelationship {
|
||||
|
||||
/**
|
||||
* The type of the lineage
|
||||
*/
|
||||
type: optional DatasetLineageType
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
namespace com.linkedin.metadata.relationship
|
||||
|
||||
/**
|
||||
* A generic model for the Evaluated-On relationship
|
||||
*/
|
||||
@pairings = [ {
|
||||
"destination" : "com.linkedin.common.urn.DatasetUrn",
|
||||
"source" : "com.linkedin.common.urn.MLModelUrn"
|
||||
}]
|
||||
record EvaluatedOn includes BaseRelationship {
|
||||
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
namespace com.linkedin.metadata.relationship
|
||||
|
||||
/**
|
||||
* A generic model for the Is-Part-Of relationship
|
||||
*/
|
||||
@pairings = [
|
||||
{
|
||||
"destination" : "com.linkedin.common.urn.DataFlowUrn",
|
||||
"source" : "com.linkedin.common.urn.DataJobUrn"
|
||||
} ]
|
||||
record IsPartOf includes BaseRelationship {
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
namespace com.linkedin.metadata.relationship
|
||||
|
||||
import com.linkedin.common.OwnershipType
|
||||
|
||||
/**
|
||||
* A generic model for the Owned-By relationship
|
||||
*/
|
||||
@pairings = [ {
|
||||
"destination" : "com.linkedin.common.urn.CorpuserUrn",
|
||||
"source" : "com.linkedin.common.urn.DatasetUrn"
|
||||
}, {
|
||||
"destination" : "com.linkedin.common.urn.CorpuserUrn",
|
||||
"source" : "com.linkedin.common.urn.DataProcessUrn"
|
||||
}, {
|
||||
"destination" : "com.linkedin.common.urn.CorpuserUrn",
|
||||
"source" : "com.linkedin.common.urn.MLModelUrn"
|
||||
}, {
|
||||
"destination" : "com.linkedin.common.urn.CorpuserUrn",
|
||||
"source" : "com.linkedin.common.urn.DataJobUrn"
|
||||
}, {
|
||||
"destination" : "com.linkedin.common.urn.CorpuserUrn",
|
||||
"source" : "com.linkedin.common.urn.DataFlowUrn"
|
||||
}, {
|
||||
"destination" : "com.linkedin.common.urn.CorpuserUrn",
|
||||
"source" : "com.linkedin.common.urn.GlossaryTermUrn"
|
||||
}, {
|
||||
"destination" : "com.linkedin.common.urn.CorpuserUrn",
|
||||
"source" : "com.linkedin.common.urn.GlossaryNodeUrn"
|
||||
} ]
|
||||
record OwnedBy includes BaseRelationship {
|
||||
|
||||
/**
|
||||
* The type of the ownership
|
||||
*/
|
||||
type: OwnershipType
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
namespace com.linkedin.metadata.relationship
|
||||
|
||||
/**
|
||||
* A generic model for the
|
||||
*/
|
||||
@pairings = [ {
|
||||
"destination" : "com.linkedin.common.urn.DatasetUrn",
|
||||
"source" : "com.linkedin.common.urn.DataJobUrn"
|
||||
} ]
|
||||
record Produces includes BaseRelationship {
|
||||
}
|
||||
@ -1,6 +0,0 @@
|
||||
namespace com.linkedin.metadata.relationship
|
||||
|
||||
/**
|
||||
* A union of all supported relationship types.
|
||||
*/
|
||||
typeref Relationship = union[Contains, IsPartOf, OwnedBy, Consumes, Produces]
|
||||
@ -1,11 +0,0 @@
|
||||
namespace com.linkedin.metadata.relationship
|
||||
|
||||
/**
|
||||
* A generic model for the Reports-To relationship
|
||||
*/
|
||||
@pairings = [ {
|
||||
"destination" : "com.linkedin.common.urn.CorpuserUrn",
|
||||
"source" : "com.linkedin.common.urn.CorpuserUrn"
|
||||
} ]
|
||||
record ReportsTo includes BaseRelationship {
|
||||
}
|
||||
@ -1,11 +0,0 @@
|
||||
namespace com.linkedin.metadata.relationship
|
||||
|
||||
/**
|
||||
* A generic model for the
|
||||
*/
|
||||
@pairings = [ {
|
||||
"destination" : "com.linkedin.common.urn.DataJobUrn",
|
||||
"source" : "com.linkedin.common.urn.DataJobUrn"
|
||||
} ]
|
||||
record RunsBefore includes BaseRelationship {
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
namespace com.linkedin.metadata.relationship
|
||||
|
||||
/**
|
||||
* A generic model for the Trained-On relationship
|
||||
*/
|
||||
@pairings = [ {
|
||||
"destination" : "com.linkedin.common.urn.DatasetUrn",
|
||||
"source" : "com.linkedin.common.urn.MLModelUrn"
|
||||
}]
|
||||
record TrainedOn includes BaseRelationship {
|
||||
|
||||
}
|
||||
@ -1,17 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
/**
|
||||
* Common fields that may apply to all documents
|
||||
*/
|
||||
record BaseDocument {
|
||||
|
||||
/**
|
||||
* Whether the entity has been removed or not
|
||||
*/
|
||||
removed: optional boolean = false
|
||||
|
||||
/**
|
||||
* All paths representing the hierarchy of this entity. This is essential for browsing various paths leading to this entity.
|
||||
*/
|
||||
browsePaths: optional array[string]
|
||||
}
|
||||
@ -1,57 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
import com.linkedin.chart.ChartQueryType
|
||||
import com.linkedin.chart.ChartType
|
||||
import com.linkedin.common.AccessLevel
|
||||
import com.linkedin.common.ChartUrn
|
||||
|
||||
/**
|
||||
* Data model for Chart entity search
|
||||
*/
|
||||
record ChartDocument includes BaseDocument {
|
||||
|
||||
/**
|
||||
* Urn for the Chart
|
||||
*/
|
||||
urn: ChartUrn
|
||||
|
||||
/**
|
||||
* Title of the chart
|
||||
*/
|
||||
title: optional string
|
||||
|
||||
/**
|
||||
* Detailed description about the chart
|
||||
*/
|
||||
description: optional string
|
||||
|
||||
/**
|
||||
* Dashboard tool ex: Looker, Redash
|
||||
*/
|
||||
tool: optional string
|
||||
|
||||
/**
|
||||
* Chart query type
|
||||
*/
|
||||
queryType: optional ChartQueryType
|
||||
|
||||
/**
|
||||
* LDAP usernames of corp users who are the owners of this chart
|
||||
*/
|
||||
owners: optional array[string]
|
||||
|
||||
/**
|
||||
* Type of the chart
|
||||
*/
|
||||
type: optional ChartType
|
||||
|
||||
/**
|
||||
* Access level for the chart
|
||||
*/
|
||||
access: optional AccessLevel
|
||||
|
||||
/**
|
||||
* List of tags for this dataset
|
||||
*/
|
||||
tags: optional array[string]
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
import com.linkedin.common.CorpGroupUrn
|
||||
|
||||
/**
|
||||
* Data model for Corp Group entity search
|
||||
*/
|
||||
record CorpGroupDocument includes BaseDocument {
|
||||
|
||||
/**
|
||||
* Urn for the Corp group.
|
||||
*/
|
||||
urn: CorpGroupUrn
|
||||
|
||||
/**
|
||||
* Email of the corp group
|
||||
*/
|
||||
email: optional string
|
||||
|
||||
/**
|
||||
* ldap usernames of corp users who are direct members of this group
|
||||
*/
|
||||
members: optional array[string]
|
||||
|
||||
/**
|
||||
* ldap usernames of corp users who are direct admins of this group
|
||||
*/
|
||||
admins: optional array[string]
|
||||
|
||||
/**
|
||||
* List of group names who are part of this group
|
||||
*/
|
||||
groups: optional array[string]
|
||||
}
|
||||
@ -1,59 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
import com.linkedin.common.CorpuserUrn
|
||||
|
||||
/**
|
||||
* Data model for CorpUserInfo entity search
|
||||
*/
|
||||
record CorpUserInfoDocument includes BaseDocument {
|
||||
|
||||
/**
|
||||
* Urn for the CorpUser.
|
||||
*/
|
||||
urn: CorpuserUrn
|
||||
|
||||
/**
|
||||
* ldap of the CorpUser
|
||||
*/
|
||||
ldap: optional string
|
||||
|
||||
/**
|
||||
* title of the CorpUser
|
||||
*/
|
||||
title: optional string
|
||||
|
||||
/**
|
||||
* direct manager's ldap of the CorpUser
|
||||
*/
|
||||
managerLdap: optional string
|
||||
|
||||
/**
|
||||
* Common name of the CorpUser, format is firstName + lastName (split by a whitespace)
|
||||
*/
|
||||
fullName: optional string
|
||||
|
||||
/**
|
||||
* About me section of the user
|
||||
*/
|
||||
aboutMe: optional string
|
||||
|
||||
/**
|
||||
* Teams that the user belongs to e.g. Metadata
|
||||
*/
|
||||
teams: optional array[string]
|
||||
|
||||
/**
|
||||
* Skills that the user possesses e.g. Machine Learning
|
||||
*/
|
||||
skills: optional array[string]
|
||||
|
||||
/**
|
||||
* Whether the corpUser is active, ref: https://iwww.corp.linkedin.com/wiki/cf/display/GTSD/Accessing+Active+Directory+via+LDAP+tools
|
||||
*/
|
||||
active: optional boolean
|
||||
|
||||
/**
|
||||
* The user's full email(s).
|
||||
*/
|
||||
emails: optional array[string]
|
||||
}
|
||||
@ -1,45 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
import com.linkedin.common.AccessLevel
|
||||
import com.linkedin.common.DashboardUrn
|
||||
|
||||
/**
|
||||
* Data model for Chart entity search
|
||||
*/
|
||||
record DashboardDocument includes BaseDocument {
|
||||
|
||||
/**
|
||||
* Urn for the Dashboard
|
||||
*/
|
||||
urn: DashboardUrn
|
||||
|
||||
/**
|
||||
* Title of the dashboard
|
||||
*/
|
||||
title: optional string
|
||||
|
||||
/**
|
||||
* Detailed description about the dashboard
|
||||
*/
|
||||
description: optional string
|
||||
|
||||
/**
|
||||
* Dashboard tool ex: Looker, Redash
|
||||
*/
|
||||
tool: optional string
|
||||
|
||||
/**
|
||||
* LDAP usernames of corp users who are the owners of this dashboard
|
||||
*/
|
||||
owners: optional array[string]
|
||||
|
||||
/**
|
||||
* Access level for the dashboard
|
||||
*/
|
||||
access: optional AccessLevel
|
||||
|
||||
/**
|
||||
* List of tags for this dataset
|
||||
*/
|
||||
tags: optional array[string]
|
||||
}
|
||||
@ -1,60 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
import com.linkedin.common.AccessLevel
|
||||
import com.linkedin.common.DataFlowUrn
|
||||
|
||||
/**
|
||||
* Data model for DataFlow entity search
|
||||
*/
|
||||
record DataFlowDocument includes BaseDocument {
|
||||
|
||||
/**
|
||||
* Urn for the DataFlow
|
||||
*/
|
||||
urn: DataFlowUrn
|
||||
|
||||
/**
|
||||
* Id of the flow
|
||||
*/
|
||||
flowId: optional string
|
||||
|
||||
/**
|
||||
* Name of the flow
|
||||
*/
|
||||
name: optional string
|
||||
|
||||
/**
|
||||
* Description of the flow
|
||||
*/
|
||||
description: optional string
|
||||
|
||||
/**
|
||||
* Workflow orchestrator ex: Azkaban, Airflow
|
||||
*/
|
||||
orchestrator: optional string
|
||||
|
||||
/**
|
||||
* Cluster of the flow
|
||||
*/
|
||||
cluster: optional string
|
||||
|
||||
/**
|
||||
* Project of the flow
|
||||
*/
|
||||
project: optional string
|
||||
|
||||
/**
|
||||
* LDAP usernames of corp users who are the owners of this flow
|
||||
*/
|
||||
owners: optional array[string]
|
||||
|
||||
/**
|
||||
* Flag to indicate if the flow has non empty corp users as owners or not.
|
||||
*/
|
||||
hasOwners: optional boolean
|
||||
|
||||
/**
|
||||
* List of tags for this dataset
|
||||
*/
|
||||
tags: optional array[string]
|
||||
}
|
||||
@ -1,77 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
import com.linkedin.common.AccessLevel
|
||||
import com.linkedin.common.DataJobUrn
|
||||
import com.linkedin.common.DatasetUrn
|
||||
|
||||
|
||||
/**
|
||||
* Data model for DataJob entity search
|
||||
*/
|
||||
record DataJobDocument includes BaseDocument {
|
||||
|
||||
/*
|
||||
* Urn for the DataJob
|
||||
*/
|
||||
urn: DataJobUrn
|
||||
|
||||
/**
|
||||
* Optional description of the job
|
||||
*/
|
||||
description: optional string
|
||||
|
||||
/**
|
||||
* Optional name of the job
|
||||
*/
|
||||
name: optional string
|
||||
|
||||
/**
|
||||
* Name of the associated data flow
|
||||
*/
|
||||
dataFlow: optional string
|
||||
|
||||
/**
|
||||
* Id of the job
|
||||
*/
|
||||
jobId: optional string
|
||||
|
||||
/**
|
||||
* LDAP usernames of corp users who are the owners of this job
|
||||
*/
|
||||
owners: optional array[string]
|
||||
|
||||
/**
|
||||
* Flag to indicate if the job has non empty corp users as owners or not.
|
||||
*/
|
||||
hasOwners: optional boolean
|
||||
|
||||
/**
|
||||
* Lineage information represented by the number of immediate input datasets of this job.
|
||||
*/
|
||||
numInputDatasets: optional long
|
||||
|
||||
/**
|
||||
* Lineage information represented by the number of immediate output datasets of this job.
|
||||
*/
|
||||
numOutputDatasets: optional long
|
||||
|
||||
/**
|
||||
* List of inputs for this job
|
||||
*/
|
||||
inputs: optional array[DatasetUrn]
|
||||
|
||||
/**
|
||||
* List of outputs for this job
|
||||
*/
|
||||
outputs: optional array[DatasetUrn]
|
||||
|
||||
/**
|
||||
* Workflow orchestrator ex: Azkaban, Airflow
|
||||
*/
|
||||
orchestrator: optional string
|
||||
|
||||
/**
|
||||
* List of tags for this dataset
|
||||
*/
|
||||
tags: optional array[string]
|
||||
}
|
||||
@ -1,61 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
import com.linkedin.common.DataProcessUrn
|
||||
import com.linkedin.common.DatasetUrn
|
||||
import com.linkedin.common.FabricType
|
||||
|
||||
/**
|
||||
* Data model for data process entity search
|
||||
*/
|
||||
record DataProcessDocument includes BaseDocument {
|
||||
|
||||
/**
|
||||
* Urn for the data process
|
||||
*/
|
||||
urn: DataProcessUrn
|
||||
|
||||
/**
|
||||
* Process native name e.g. a ETL script name
|
||||
*/
|
||||
name: optional string
|
||||
|
||||
/**
|
||||
* Orchestrator name for this process, such as Azure Data Factory
|
||||
*/
|
||||
orchestrator: optional string
|
||||
|
||||
/**
|
||||
* Fabric type where data process belongs to or where it was generated
|
||||
*/
|
||||
origin: optional FabricType
|
||||
|
||||
/**
|
||||
* LDAP usernames of corp users who are the owners of this process
|
||||
*/
|
||||
owners: optional array[string]
|
||||
|
||||
/**
|
||||
* Flag to indicate if the process has non empty corp users as owners or not.
|
||||
*/
|
||||
hasOwners: optional boolean
|
||||
|
||||
/**
|
||||
* Lineage information represented by the number of immediate input datasets of this process.
|
||||
*/
|
||||
numInputDatasets: optional long
|
||||
|
||||
/**
|
||||
* Lineage information represented by the number of immediate output datasets of this process.
|
||||
*/
|
||||
numOutputDatasets: optional long
|
||||
|
||||
/**
|
||||
* List of inputs for this process
|
||||
*/
|
||||
inputs: optional array[DatasetUrn]
|
||||
|
||||
/**
|
||||
* List of outputs for this process
|
||||
*/
|
||||
outputs: optional array[DatasetUrn]
|
||||
}
|
||||
@ -1,100 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
import com.linkedin.common.DatasetUrn
|
||||
import com.linkedin.common.FabricType
|
||||
|
||||
/**
|
||||
* Data model for dataset entity search
|
||||
*/
|
||||
record DatasetDocument includes BaseDocument {
|
||||
|
||||
/**
|
||||
* Urn for the dataset
|
||||
*/
|
||||
urn: DatasetUrn
|
||||
|
||||
/**
|
||||
* Dataset native name e.g. {db}.{table}, /dir/subdir/{name}, or {name}
|
||||
*/
|
||||
name: optional string
|
||||
|
||||
/**
|
||||
* Platform name for the dataset
|
||||
*/
|
||||
platform: optional string
|
||||
|
||||
/**
|
||||
* Fabric type where dataset belongs to or where it was generated
|
||||
*/
|
||||
origin: optional FabricType
|
||||
|
||||
/**
|
||||
* LDAP usernames of corp users who are the owners of this dataset
|
||||
*/
|
||||
owners: optional array[string]
|
||||
|
||||
/**
|
||||
* Flag to indicate if the dataset is deprecated.
|
||||
*/
|
||||
deprecated: optional boolean
|
||||
|
||||
/**
|
||||
* Documentation of the dataset.
|
||||
*/
|
||||
description: optional string
|
||||
|
||||
/**
|
||||
* Field paths of the dataset
|
||||
*/
|
||||
fieldPaths: optional array[string]
|
||||
|
||||
/**
|
||||
* Flag to indicate if the dataset has non empty corp users as owners or not.
|
||||
*/
|
||||
hasOwners: optional boolean
|
||||
|
||||
/**
|
||||
* Flag to indicate if the dataset has non-empty schema or not.
|
||||
*/
|
||||
hasSchema: optional boolean
|
||||
|
||||
/**
|
||||
* Lineage information represented by the number of immediate downstream datasets of this dataset.
|
||||
*/
|
||||
numDownstreamDatasets: optional long
|
||||
|
||||
/**
|
||||
* List of upstreams for this dataset
|
||||
*/
|
||||
upstreams: optional array[DatasetUrn]
|
||||
|
||||
/**
|
||||
* List of tags for this dataset
|
||||
*/
|
||||
tags: optional array[string]
|
||||
|
||||
/**
|
||||
* List of field descriptions
|
||||
*/
|
||||
fieldDescriptions: optional array[string]
|
||||
|
||||
/**
|
||||
* List of tags applied to fields
|
||||
*/
|
||||
fieldTags: optional array[string]
|
||||
|
||||
/**
|
||||
* List of field descriptions
|
||||
*/
|
||||
editedFieldDescriptions: optional array[string]
|
||||
|
||||
/**
|
||||
* List of tags applied to fields
|
||||
*/
|
||||
editedFieldTags: optional array[string]
|
||||
|
||||
/**
|
||||
* List of terms for this dataset
|
||||
*/
|
||||
glossaryTerms: optional array[string]
|
||||
}
|
||||
@ -1,6 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
/**
|
||||
* A union of all supported document types.
|
||||
*/
|
||||
typeref Document = union[CorpUserInfoDocument, DatasetDocument]
|
||||
@ -1,29 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
import com.linkedin.common.GlossaryNodeUrn
|
||||
|
||||
/**
|
||||
* Data model for GlossaryNodeInfo entity search
|
||||
*/
|
||||
record GlossaryNodeInfoDocument includes BaseDocument {
|
||||
|
||||
/**
|
||||
* Urn for the GlossaryNode.
|
||||
*/
|
||||
urn: GlossaryNodeUrn
|
||||
|
||||
/**
|
||||
* Name of business node
|
||||
*/
|
||||
name: optional string
|
||||
|
||||
/**
|
||||
* Definition of business node
|
||||
*/
|
||||
definition: optional string
|
||||
|
||||
/**
|
||||
* LDAP usernames of corp users who are the owners of this business node
|
||||
*/
|
||||
owners: optional array[string]
|
||||
}
|
||||
@ -1,39 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
import com.linkedin.common.GlossaryTermUrn
|
||||
|
||||
/**
|
||||
* Data model for GlossaryTermInfo entity search
|
||||
*/
|
||||
record GlossaryTermInfoDocument includes BaseDocument {
|
||||
|
||||
/**
|
||||
* Urn for the GlossaryTerm.
|
||||
*/
|
||||
urn: GlossaryTermUrn
|
||||
|
||||
/**
|
||||
* Name of business term
|
||||
*/
|
||||
name: optional string
|
||||
|
||||
/**
|
||||
* Definition of business term
|
||||
*/
|
||||
definition: optional string
|
||||
|
||||
/**
|
||||
* LDAP usernames of corp users who are the owners of this dataset
|
||||
*/
|
||||
owners: optional array[string]
|
||||
|
||||
/**
|
||||
* Source of the Business Term (INTERNAL or EXTERNAL) with default value as INTERNAL
|
||||
*/
|
||||
termSource: optional string
|
||||
|
||||
/**
|
||||
* External Reference to the business-term (URL)
|
||||
*/
|
||||
sourceRef: optional string
|
||||
}
|
||||
@ -1,71 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
import com.linkedin.common.MLModelUrn
|
||||
import com.linkedin.common.DatasetUrn
|
||||
import com.linkedin.common.FabricType
|
||||
|
||||
/**
|
||||
* Data model for ML Model entity search
|
||||
*/
|
||||
record MLModelDocument includes BaseDocument {
|
||||
|
||||
/**
|
||||
* Urn for the model
|
||||
*/
|
||||
urn: MLModelUrn
|
||||
|
||||
/**
|
||||
* Name of model
|
||||
*/
|
||||
name: optional string
|
||||
|
||||
/**
|
||||
* Platform name for the model
|
||||
*/
|
||||
platform: optional string
|
||||
|
||||
/**
|
||||
* Fabric type where model belongs to or where it was generated
|
||||
*/
|
||||
origin: optional FabricType
|
||||
|
||||
/**
|
||||
* Description of the model
|
||||
*/
|
||||
description: optional string
|
||||
|
||||
/**
|
||||
* Timestamp model was created
|
||||
*/
|
||||
createdTimestamp: optional long
|
||||
|
||||
/**
|
||||
* Whether or not the Model has owners
|
||||
*/
|
||||
hasOwners: optional boolean
|
||||
|
||||
/**
|
||||
* LDAP usernames of corp users who are the owners of this model
|
||||
*/
|
||||
owners: optional array[string]
|
||||
|
||||
/**
|
||||
* Type of Algorithm or Model such as whether it is a Naive Bayes classifier, Convolutional Neural Network, etc
|
||||
*/
|
||||
type: optional string
|
||||
|
||||
/**
|
||||
* What datasets were used to train the model?
|
||||
*/
|
||||
trainingDatasets: optional array[DatasetUrn]
|
||||
|
||||
/**
|
||||
* What datasets were used to evaluate the model?
|
||||
*/
|
||||
evaluationDatasets: optional array[DatasetUrn]
|
||||
|
||||
/**
|
||||
* Whether or not the model is currently active
|
||||
*/
|
||||
active: optional boolean
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
namespace com.linkedin.metadata.search
|
||||
|
||||
import com.linkedin.common.TagUrn
|
||||
|
||||
/**
|
||||
* Data model for tag entity search
|
||||
*/
|
||||
record TagDocument includes BaseDocument {
|
||||
|
||||
/**
|
||||
* Urn for the dataset
|
||||
*/
|
||||
urn: TagUrn
|
||||
|
||||
/**
|
||||
* Tag name e.g. `Legacy`
|
||||
*/
|
||||
name: optional string
|
||||
}
|
||||
@ -1,31 +0,0 @@
|
||||
namespace com.linkedin.mxe
|
||||
|
||||
import com.linkedin.avro2pegasus.events.KafkaAuditHeader
|
||||
import com.linkedin.metadata.entity.Entity
|
||||
import com.linkedin.metadata.relationship.Relationship
|
||||
|
||||
/**
|
||||
* Kafka event for capturing update made to a list of entities and relationships.
|
||||
*/
|
||||
record MetadataGraphEvent {
|
||||
|
||||
/**
|
||||
* Kafka audit header. See go/kafkaauditheader for more info.
|
||||
*/
|
||||
auditHeader: optional KafkaAuditHeader
|
||||
|
||||
/**
|
||||
* A list of entity updates-or-inserts. Only fields updated are set in the case of partial update.
|
||||
*/
|
||||
upsertedEntities: array[Entity]
|
||||
|
||||
/**
|
||||
* A list of removed relationships. Only fields used to identify the relationships to remove are set.
|
||||
*/
|
||||
removedRelationships: array[Relationship]
|
||||
|
||||
/**
|
||||
* A list of relationship updates-or-inserts.
|
||||
*/
|
||||
upsertedRelationships: array[Relationship]
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
namespace com.linkedin.mxe
|
||||
|
||||
import com.linkedin.avro2pegasus.events.KafkaAuditHeader
|
||||
import com.linkedin.metadata.search.Document
|
||||
|
||||
/**
|
||||
* Kafka event for capturing update made to a list of search documents.
|
||||
*/
|
||||
record MetadataSearchEvent {
|
||||
|
||||
/**
|
||||
* Kafka audit header. See go/kafkaauditheader for more info.
|
||||
*/
|
||||
auditHeader: optional KafkaAuditHeader
|
||||
|
||||
/**
|
||||
* A list of search document updates-or-inserts. Only fields updated are set in the case of partial update.
|
||||
*/
|
||||
upsertedDocuments: array[Document]
|
||||
}
|
||||
@ -5,8 +5,6 @@ import com.linkedin.data.template.RecordTemplate;
|
||||
import com.linkedin.data.template.UnionTemplate;
|
||||
import com.linkedin.metadata.validator.AspectValidator;
|
||||
import com.linkedin.metadata.validator.DeltaValidator;
|
||||
import com.linkedin.metadata.validator.EntityValidator;
|
||||
import com.linkedin.metadata.validator.RelationshipValidator;
|
||||
import com.linkedin.metadata.validator.SnapshotValidator;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
@ -15,30 +13,14 @@ import java.util.stream.Collectors;
|
||||
import javax.annotation.Nonnull;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static com.linkedin.metadata.ModelValidationConstants.*;
|
||||
import static org.testng.AssertJUnit.*;
|
||||
import static com.linkedin.metadata.ModelValidationConstants.IGNORED_ASPECT_CLASSES;
|
||||
import static com.linkedin.metadata.ModelValidationConstants.IGNORED_DELTA_CLASSES;
|
||||
import static com.linkedin.metadata.ModelValidationConstants.IGNORED_SNAPSHOT_CLASSES;
|
||||
import static org.testng.AssertJUnit.assertFalse;
|
||||
|
||||
|
||||
public class ModelValidation {
|
||||
|
||||
@Test
|
||||
public void validateEntities() throws Exception {
|
||||
List<? extends Class<? extends RecordTemplate>> entities =
|
||||
getRecordTemplatesInPackage("com.linkedin.metadata.entity", IGNORED_ENTITY_CLASSES);
|
||||
|
||||
assertFalse("Failed to find any entities", entities.isEmpty());
|
||||
entities.forEach(EntityValidator::validateEntitySchema);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void validateRelationships() throws Exception {
|
||||
List<? extends Class<? extends RecordTemplate>> relationships =
|
||||
getRecordTemplatesInPackage("com.linkedin.metadata.relationship", IGNORED_RELATIONSHIP_CLASSES);
|
||||
|
||||
assertFalse("Failed to find any relationships", relationships.isEmpty());
|
||||
relationships.forEach(RelationshipValidator::validateRelationshipSchema);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void validateAspects() throws Exception {
|
||||
List<? extends Class<? extends UnionTemplate>> aspects =
|
||||
@ -53,7 +35,6 @@ public class ModelValidation {
|
||||
List<? extends Class<? extends RecordTemplate>> snapshots =
|
||||
getRecordTemplatesInPackage("com.linkedin.metadata.snapshot", IGNORED_SNAPSHOT_CLASSES);
|
||||
|
||||
|
||||
assertFalse("Failed to find any snapshots", snapshots.isEmpty());
|
||||
snapshots.forEach(SnapshotValidator::validateSnapshotSchema);
|
||||
}
|
||||
|
||||
@ -3,9 +3,6 @@ package com.linkedin.metadata;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.linkedin.data.template.RecordTemplate;
|
||||
import com.linkedin.data.template.UnionTemplate;
|
||||
import com.linkedin.metadata.entity.BaseEntity;
|
||||
import com.linkedin.metadata.relationship.BaseRelationship;
|
||||
import com.linkedin.metadata.search.BaseDocument;
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
@ -15,13 +12,6 @@ public class ModelValidationConstants {
|
||||
// Util class
|
||||
}
|
||||
|
||||
static final Set<Class<? extends RecordTemplate>> IGNORED_ENTITY_CLASSES = ImmutableSet.of(BaseEntity.class);
|
||||
|
||||
static final Set<Class<? extends RecordTemplate>> IGNORED_RELATIONSHIP_CLASSES =
|
||||
ImmutableSet.of(BaseRelationship.class);
|
||||
|
||||
static final Set<Class<? extends RecordTemplate>> IGNORED_DOCUMENT_CLASSES = ImmutableSet.of(BaseDocument.class);
|
||||
|
||||
static final Set<Class<? extends UnionTemplate>> IGNORED_ASPECT_CLASSES = ImmutableSet.of();
|
||||
|
||||
static final Set<Class<? extends RecordTemplate>> IGNORED_SNAPSHOT_CLASSES = ImmutableSet.of();
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user