yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-03 14:23:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingestion): ingest emails as empty if no ldap attribute (#9433)

Browse Source 
Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
This commit is contained in:
Tim Drahn 2024-07-01 02:23:41 -04:00 committed by GitHub
parent 7fb56ef717
commit 93616f7869
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 56 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
metadata-ingestion/src/datahub/ingestion/source/ldap.py
View File

@ -334,10 +334,12 @@ class LDAPSource(StatefulIngestionSourceBase):
manager_ldap = guess_person_ldap(m_attrs, self.config, self.report) manager_ldap = guess_person_ldap(m_attrs, self.config, self.report)
m_email = get_attr_or_none( m_email = get_attr_or_none(
m_attrs, self.config.user_attrs_map["email"], manager_ldap m_attrs, self.config.user_attrs_map["email"]
) )
make_manager_urn = ( make_manager_urn = (
m_email if self.config.use_email_as_username else manager_ldap m_email
if m_email and self.config.use_email_as_username
else manager_ldap
) )
except ldap.LDAPError as e: except ldap.LDAPError as e:
@ -377,7 +379,7 @@ class LDAPSource(StatefulIngestionSourceBase):
last_name = attrs[self.config.user_attrs_map["lastName"]][0].decode() last_name = attrs[self.config.user_attrs_map["lastName"]][0].decode()
groups = parse_groups(attrs, self.config.user_attrs_map["memberOf"]) groups = parse_groups(attrs, self.config.user_attrs_map["memberOf"])
email = get_attr_or_none(attrs, self.config.user_attrs_map["email"], ldap_user) email = get_attr_or_none(attrs, self.config.user_attrs_map["email"])
display_name = get_attr_or_none( display_name = get_attr_or_none(
attrs, self.config.user_attrs_map["displayName"], full_name attrs, self.config.user_attrs_map["displayName"], full_name
) )
@ -404,7 +406,9 @@ class LDAPSource(StatefulIngestionSourceBase):
manager_urn = f"urn:li:corpuser:{manager_ldap}" if manager_ldap else None manager_urn = f"urn:li:corpuser:{manager_ldap}" if manager_ldap else None
make_user_urn = email if self.config.use_email_as_username else ldap_user make_user_urn = (
email if email and self.config.use_email_as_username else ldap_user
)
user_snapshot = CorpUserSnapshotClass( user_snapshot = CorpUserSnapshotClass(
urn=f"urn:li:corpuser:{make_user_urn}", urn=f"urn:li:corpuser:{make_user_urn}",
@ -438,9 +442,7 @@ class LDAPSource(StatefulIngestionSourceBase):
admins = parse_users(attrs, self.config.group_attrs_map["admins"]) admins = parse_users(attrs, self.config.group_attrs_map["admins"])
members = parse_users(attrs, self.config.group_attrs_map["members"]) members = parse_users(attrs, self.config.group_attrs_map["members"])
email = get_attr_or_none( email = get_attr_or_none(attrs, self.config.group_attrs_map["email"])
attrs, self.config.group_attrs_map["email"], full_name
)
description = get_attr_or_none( description = get_attr_or_none(
attrs, self.config.group_attrs_map["description"] attrs, self.config.group_attrs_map["description"]
) )
@ -448,7 +450,9 @@ class LDAPSource(StatefulIngestionSourceBase):
attrs, self.config.group_attrs_map["displayName"] attrs, self.config.group_attrs_map["displayName"]
) )
make_group_urn = email if self.config.use_email_as_username else full_name make_group_urn = (
email if email and self.config.use_email_as_username else full_name
)
group_snapshot = CorpGroupSnapshotClass( group_snapshot = CorpGroupSnapshotClass(
urn=f"urn:li:corpGroup:{make_group_urn}", urn=f"urn:li:corpGroup:{make_group_urn}",

62
metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json
View File

@ -6,7 +6,6 @@
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"email": "simpons-group",
"admins": [], "admins": [],
"members": [ "members": [
"urn:li:corpuser:hsimpson", "urn:li:corpuser:hsimpson",
@ -20,7 +19,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -35,7 +35,6 @@
}, },
"active": true, "active": true,
"displayName": "Bart Simpson", "displayName": "Bart Simpson",
"email": "bsimpson",
"title": "Mr. Boss", "title": "Mr. Boss",
"firstName": "Bart", "firstName": "Bart",
"lastName": "Simpson", "lastName": "Simpson",
@ -52,7 +51,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -87,7 +87,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -102,7 +103,6 @@
}, },
"active": true, "active": true,
"displayName": "Lisa Simpson", "displayName": "Lisa Simpson",
"email": "lsimpson",
"firstName": "Lisa", "firstName": "Lisa",
"lastName": "Simpson", "lastName": "Simpson",
"fullName": "Lisa Simpson" "fullName": "Lisa Simpson"
@ -118,7 +118,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -133,7 +134,6 @@
}, },
"active": true, "active": true,
"displayName": "Maggie Simpson", "displayName": "Maggie Simpson",
"email": "msimpson",
"firstName": "Maggie", "firstName": "Maggie",
"lastName": "Simpson", "lastName": "Simpson",
"fullName": "Maggie Simpson" "fullName": "Maggie Simpson"
@ -149,7 +149,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -164,7 +165,6 @@
}, },
"active": true, "active": true,
"displayName": "Hester Bevan", "displayName": "Hester Bevan",
"email": "hbevan",
"firstName": "Hester", "firstName": "Hester",
"lastName": "Bevan", "lastName": "Bevan",
"fullName": "Hester Bevan" "fullName": "Hester Bevan"
@ -180,7 +180,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -195,7 +196,6 @@
}, },
"active": true, "active": true,
"displayName": "Evalyn Haas", "displayName": "Evalyn Haas",
"email": "ehaas",
"firstName": "Evalyn", "firstName": "Evalyn",
"lastName": "Haas", "lastName": "Haas",
"fullName": "Evalyn Haas" "fullName": "Evalyn Haas"
@ -211,7 +211,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -221,7 +222,6 @@
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"email": "HR Department",
"admins": [], "admins": [],
"members": [], "members": [],
"groups": [] "groups": []
@ -232,7 +232,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -242,7 +243,6 @@
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"email": "Finance Department",
"admins": [], "admins": [],
"members": [], "members": [],
"groups": [] "groups": []
@ -253,7 +253,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -268,7 +269,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -283,7 +285,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -298,7 +301,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -313,7 +317,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -328,7 +333,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -343,7 +349,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -358,7 +365,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -373,7 +381,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -388,7 +397,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
} }
] ]

1
metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_group_stateful.json
View File

@ -6,7 +6,6 @@
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"email": "HR Department",
"admins": [], "admins": [],
"members": [], "members": [],
"groups": [] "groups": []

1
metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_stateful.json
View File

@ -9,7 +9,6 @@
"customProperties": {}, "customProperties": {},
"active": true, "active": true,
"displayName": "Bart Simpson", "displayName": "Bart Simpson",
"email": "bsimpson",
"title": "Mr. Boss", "title": "Mr. Boss",
"firstName": "Bart", "firstName": "Bart",
"lastName": "Simpson", "lastName": "Simpson",

2
metadata-ingestion/tests/integration/ldap/ldap_mces_golden_group_stateful.json
View File

@ -6,7 +6,6 @@
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"email": "HR Department",
"admins": [], "admins": [],
"members": [], "members": [],
"groups": [] "groups": []
@ -29,7 +28,6 @@
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.identity.CorpGroupInfo": { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": {
"email": "Finance Department",
"admins": [], "admins": [],
"members": [], "members": [],
"groups": [] "groups": []

1
metadata-ingestion/tests/integration/ldap/ldap_mces_golden_stateful.json
View File

@ -9,7 +9,6 @@
"customProperties": {}, "customProperties": {},
"active": true, "active": true,
"displayName": "Bart Simpson", "displayName": "Bart Simpson",
"email": "bsimpson",
"title": "Mr. Boss", "title": "Mr. Boss",
"firstName": "Bart", "firstName": "Bart",
"lastName": "Simpson", "lastName": "Simpson",

14
metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json
View File

@ -9,7 +9,6 @@
"customProperties": {}, "customProperties": {},
"active": true, "active": true,
"displayName": "Hester Bevan", "displayName": "Hester Bevan",
"email": "hbevan",
"firstName": "Hester", "firstName": "Hester",
"lastName": "Bevan", "lastName": "Bevan",
"fullName": "Hester Bevan" "fullName": "Hester Bevan"
@ -28,7 +27,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -41,7 +41,6 @@
"customProperties": {}, "customProperties": {},
"active": true, "active": true,
"displayName": "Evalyn Haas", "displayName": "Evalyn Haas",
"email": "ehaas",
"firstName": "Evalyn", "firstName": "Evalyn",
"lastName": "Haas", "lastName": "Haas",
"fullName": "Evalyn Haas" "fullName": "Evalyn Haas"
@ -60,7 +59,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -75,7 +75,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
}, },
{ {
@ -90,7 +91,8 @@
}, },
"systemMetadata": { "systemMetadata": {
"lastObserved": 1615443388097, "lastObserved": 1615443388097,
"runId": "ldap-test" "runId": "ldap-test",
"lastRunId": "no-run-id-provided"
} }
} }
] ]