yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-31 10:49:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/transformer): avoid duplicating terms (#10348)

Browse Source
This commit is contained in:
dushayntAW 2024-04-22 20:15:58 +05:30 committed by GitHub
parent 08731055ba
commit 3668a56df7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_schema_terms.py
View File

@ -71,6 +71,15 @@ class AddDatasetSchemaTerms(DatasetSchemaMetadataTransformer):
if len(terms_to_add) == 0: if len(terms_to_add) == 0:
terms_to_add = all_terms terms_to_add = all_terms
new_glossary_terms = []
new_glossary_terms.extend(server_terms)
new_glossary_terms.extend(terms_to_add)
unique_gloseary_terms = []
for term in new_glossary_terms:
if term not in unique_gloseary_terms:
unique_gloseary_terms.append(term)
new_glossary_term = GlossaryTermsClass( new_glossary_term = GlossaryTermsClass(
terms=[], terms=[],
auditStamp=schema_field.glossaryTerms.auditStamp auditStamp=schema_field.glossaryTerms.auditStamp
@ -79,11 +88,9 @@ class AddDatasetSchemaTerms(DatasetSchemaMetadataTransformer):
time=builder.get_sys_time(), actor="urn:li:corpUser:restEmitter" time=builder.get_sys_time(), actor="urn:li:corpUser:restEmitter"
), ),
) )
new_glossary_term.terms.extend(terms_to_add) new_glossary_term.terms.extend(unique_gloseary_terms)
new_glossary_term.terms.extend(server_terms)
schema_field.glossaryTerms = new_glossary_term schema_field.glossaryTerms = new_glossary_term
return schema_field return schema_field
def transform_aspect( def transform_aspect(