yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-08 13:36:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed: SampleData Teams and teamTypes (#7682)

Browse Source
This commit is contained in:
NiharDoshi99 2022-09-26 18:22:41 +05:30 committed by GitHub
parent 23f6e18a17
commit d9f3ab23d7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 1633 additions and 1407 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
ingestion/examples/sample_data/teams/teams.json Normal file
View File

@ -0,0 +1,94 @@
{
"teams": [
{
"name": "Engineering",
"teamType": "BusinessUnit",
"parent": null,
"children": ["Infrastructure", "Applications", "Marketplace", "Payments"]
},
{
"name": "Infrastructure",
"teamType": "BusinessUnit",
"parent": ["Engineering"],
"children": ["Compute", "Data","DevOps"]
},
{
"name": "Compute",
"teamType": "Group",
"parent": ["Infrastructure"],
"children": null
},
{
"name": "Data",
"teamType": "Group",
"parent": ["Infrastructure"],
"children": null
},
{
"name": "DevOps",
"teamType": "Group",
"parent": ["Infrastructure"],
"children": null
},
{
"name": "Applications",
"teamType": "Group",
"parent": ["Engineering"],
"children": null
},
{
"name": "Marketplace",
"teamType": "Group",
"parent": ["Engineering"],
"children": null
},
{
"name": "Payments",
"teamType": "Group",
"parent": ["Engineering"],
"children": null
},
{
"name": "Finance",
"teamType":"BusinessUnit",
"parent": null,
"children": ["Marketing", "Sales", "Accounting"]
},
{
"name": "Marketing",
"teamType": "Group",
"parent": ["Finance"],
"children": null
},
{
"name": "Accounting",
"teamType": "Group",
"parent": ["Finance"],
"children": null
},
{
"name": "Sales",
"teamType": "Group",
"parent": ["Finance"],
"children": null
},
{
"name": "Legal",
"teamType": "BusinessUnit",
"parent": null,
"children": ["Merger & Acquisitions", "Legal Admin"]
},
{
"name": "Merger & Acquisitions",
"teamType": "Group",
"parent": ["Legal"],
"children": null
},
{
"name": "Legal Admin",
"teamType": "Group",
"parent": ["Legal"],
"children": null
}
]
}

2910
ingestion/examples/sample_data/users/users.json
View File

File diff suppressed because it is too large Load Diff

2
ingestion/src/metadata/ingestion/sink/metadata_rest.py
View File

@ -456,7 +456,7 @@ class MetadataRestSink(Sink[Entity]):
for role in record.roles:
try:
role_entity = self.metadata.get_by_name(
entity=Role, fqn=str(role.name.__root__)
entity=Role, fqn=str(role.name.__root__.__root__)
)
except APIError:
role_entity = self._create_role(role)

34
ingestion/src/metadata/ingestion/source/database/sample_data.py
View File

@ -67,7 +67,7 @@ from metadata.generated.schema.entity.services.databaseService import DatabaseSe
from metadata.generated.schema.entity.services.messagingService import MessagingService
from metadata.generated.schema.entity.services.mlmodelService import MlModelService
from metadata.generated.schema.entity.services.pipelineService import PipelineService
from metadata.generated.schema.entity.teams.team import TeamType
from metadata.generated.schema.entity.teams.team import Team, TeamType
from metadata.generated.schema.entity.teams.user import User
from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource,
@ -352,6 +352,9 @@ class SampleDataSource(Source[Entity]):
self.service_connection.sampleDataFolder + "/lineage/lineage.json", "r"
)
)
self.teams = json.load(
open(self.service_connection.sampleDataFolder + "/teams/teams.json", "r")
)
self.users = json.load(
open(self.service_connection.sampleDataFolder + "/users/users.json", "r")
)
@ -418,6 +421,7 @@ class SampleDataSource(Source[Entity]):
pass
def next_record(self) -> Iterable[Entity]:
yield from self.ingest_teams()
yield from self.ingest_users()
yield from self.ingest_locations()
yield from self.ingest_glue()
@ -434,6 +438,32 @@ class SampleDataSource(Source[Entity]):
yield from self.ingest_test_case()
yield from self.ingest_test_case_results()
def ingest_teams(self):
for team in self.teams["teams"]:
team_to_ingest = CreateTeamRequest(
name=team["name"], teamType=team["teamType"]
)
if team["parent"] != None:
parent_list_id = []
for parent in team["parent"]:
tries = 3
parent_object = self.metadata.get_by_name(entity=Team, fqn=parent)
while not parent_object and tries > 0:
logger.info(f"Trying to GET {parent} Parent Team")
parent_object = self.metadata.get_by_name(
entity=Team,
fqn=parent,
)
tries -= 1
if parent_object:
parent_list_id.append(parent_object.id)
team_to_ingest.parents = parent_list_id
yield team_to_ingest
def ingest_locations(self) -> Iterable[Location]:
for location in self.locations["locations"]:
location_ev = CreateLocationRequest(
@ -759,7 +789,7 @@ class SampleDataSource(Source[Entity]):
name=user["teams"],
displayName=user["teams"],
description=f"This is {user['teams']} description.",
teamType=TeamType.Department,
teamType=user["teamType"],
)
]
if not self.list_policies: