yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-03 14:23:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingestion): ingest emails as empty if no ldap attribute (#9433)

Browse Source 
Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
This commit is contained in:
Tim Drahn 2024-07-01 02:23:41 -04:00 committed by GitHub
parent 7fb56ef717
commit 93616f7869
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 56 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
metadata-ingestion/src/datahub/ingestion/source/ldap.py
View File

@ -334,10 +334,12 @@ class LDAPSource(StatefulIngestionSourceBase):
manager_ldap = guess_person_ldap(m_attrs, self.config, self.report)
m_email = get_attr_or_none(
m_attrs, self.config.user_attrs_map["email"], manager_ldap
m_attrs, self.config.user_attrs_map["email"]
)
make_manager_urn = (
m_email if self.config.use_email_as_username else manager_ldap
m_email
if m_email and self.config.use_email_as_username
else manager_ldap
)
except ldap.LDAPError as e:
@ -377,7 +379,7 @@ class LDAPSource(StatefulIngestionSourceBase):
last_name = attrs[self.config.user_attrs_map["lastName"]][0].decode()
groups = parse_groups(attrs, self.config.user_attrs_map["memberOf"])
email = get_attr_or_none(attrs, self.config.user_attrs_map["email"], ldap_user)
email = get_attr_or_none(attrs, self.config.user_attrs_map["email"])
display_name = get_attr_or_none(
attrs, self.config.user_attrs_map["displayName"], full_name
)
@ -404,7 +406,9 @@ class LDAPSource(StatefulIngestionSourceBase):
manager_urn = f"urn:li:corpuser:{manager_ldap}" if manager_ldap else None
make_user_urn = email if self.config.use_email_as_username else ldap_user
make_user_urn = (
email if email and self.config.use_email_as_username else ldap_user
)
user_snapshot = CorpUserSnapshotClass(
urn=f"urn:li:corpuser:{make_user_urn}",
@ -438,9 +442,7 @@ class LDAPSource(StatefulIngestionSourceBase):
admins = parse_users(attrs, self.config.group_attrs_map["admins"])
members = parse_users(attrs, self.config.group_attrs_map["members"])
email = get_attr_or_none(
attrs, self.config.group_attrs_map["email"], full_name
)
email = get_attr_or_none(attrs, self.config.group_attrs_map["email"])
description = get_attr_or_none(
attrs, self.config.group_attrs_map["description"]
)
@ -448,7 +450,9 @@ class LDAPSource(StatefulIngestionSourceBase):
attrs, self.config.group_attrs_map["displayName"]
)
make_group_urn = email if self.config.use_email_as_username else full_name
make_group_urn = (
email if email and self.config.use_email_as_username else full_name
)
group_snapshot = CorpGroupSnapshotClass(
urn=f"urn:li:corpGroup:{make_group_urn}",

62
metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json
View File

@ -6,7 +6,6 @@
"aspects": [
{
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"email": "simpons-group",
"admins": [],
"members": [
"urn:li:corpuser:hsimpson",
@ -20,7 +19,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -35,7 +35,6 @@
},
"active": true,
"displayName": "Bart Simpson",
"email": "bsimpson",
"title": "Mr. Boss",
"firstName": "Bart",
"lastName": "Simpson",
@ -52,7 +51,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -87,7 +87,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -102,7 +103,6 @@
},
"active": true,
"displayName": "Lisa Simpson",
"email": "lsimpson",
"firstName": "Lisa",
"lastName": "Simpson",
"fullName": "Lisa Simpson"
@ -118,7 +118,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -133,7 +134,6 @@
},
"active": true,
"displayName": "Maggie Simpson",
"email": "msimpson",
"firstName": "Maggie",
"lastName": "Simpson",
"fullName": "Maggie Simpson"
@ -149,7 +149,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -164,7 +165,6 @@
},
"active": true,
"displayName": "Hester Bevan",
"email": "hbevan",
"firstName": "Hester",
"lastName": "Bevan",
"fullName": "Hester Bevan"
@ -180,7 +180,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -195,7 +196,6 @@
},
"active": true,
"displayName": "Evalyn Haas",
"email": "ehaas",
"firstName": "Evalyn",
"lastName": "Haas",
"fullName": "Evalyn Haas"
@ -211,7 +211,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -221,7 +222,6 @@
"aspects": [
{
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"email": "HR Department",
"admins": [],
"members": [],
"groups": []
@ -232,7 +232,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -242,7 +243,6 @@
"aspects": [
{
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"email": "Finance Department",
"admins": [],
"members": [],
"groups": []
@ -253,7 +253,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -268,7 +269,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -283,7 +285,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -298,7 +301,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -313,7 +317,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -328,7 +333,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -343,7 +349,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -358,7 +365,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -373,7 +381,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -388,7 +397,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
}
]

1
metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_group_stateful.json
View File

@ -6,7 +6,6 @@
"aspects": [
{
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"email": "HR Department",
"admins": [],
"members": [],
"groups": []

1
metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_stateful.json
View File

@ -9,7 +9,6 @@
"customProperties": {},
"active": true,
"displayName": "Bart Simpson",
"email": "bsimpson",
"title": "Mr. Boss",
"firstName": "Bart",
"lastName": "Simpson",

2
metadata-ingestion/tests/integration/ldap/ldap_mces_golden_group_stateful.json
View File

@ -6,7 +6,6 @@
"aspects": [
{
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"email": "HR Department",
"admins": [],
"members": [],
"groups": []
@ -29,7 +28,6 @@
"aspects": [
{
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"email": "Finance Department",
"admins": [],
"members": [],
"groups": []

1
metadata-ingestion/tests/integration/ldap/ldap_mces_golden_stateful.json
View File

@ -9,7 +9,6 @@
"customProperties": {},
"active": true,
"displayName": "Bart Simpson",
"email": "bsimpson",
"title": "Mr. Boss",
"firstName": "Bart",
"lastName": "Simpson",

14
metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json
View File

@ -9,7 +9,6 @@
"customProperties": {},
"active": true,
"displayName": "Hester Bevan",
"email": "hbevan",
"firstName": "Hester",
"lastName": "Bevan",
"fullName": "Hester Bevan"
@ -28,7 +27,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -41,7 +41,6 @@
"customProperties": {},
"active": true,
"displayName": "Evalyn Haas",
"email": "ehaas",
"firstName": "Evalyn",
"lastName": "Haas",
"fullName": "Evalyn Haas"
@ -60,7 +59,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -75,7 +75,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
},
{
@ -90,7 +91,8 @@
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "ldap-test"
"runId": "ldap-test",
"lastRunId": "no-run-id-provided"
}
}
]