From cdce2ca08c8b42e6318d53392152a7d4e89904e1 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Sun, 2 Apr 2023 15:04:00 -0700 Subject: [PATCH] Fixes #10241 - Include Status, Owner and Reviewer in Bulk Glossary Upload (#10881) --- .../java/org/openmetadata/csv/CsvUtil.java | 5 +++ .../java/org/openmetadata/csv/EntityCsv.java | 6 ++-- .../service/jdbi3/GlossaryRepository.java | 31 ++++++++++++++++++- .../glossary/glossaryCsvDocumentation.json | 27 ++++++++++++++++ .../org/openmetadata/csv/EntityCsvTest.java | 6 ++-- .../glossary/GlossaryResourceTest.java | 31 +++++++++++++------ 6 files changed, 90 insertions(+), 16 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java index 6c48b2669e9..79e2c32a85f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java @@ -135,4 +135,9 @@ public final class CsvUtil { nullOrEmpty(tags) ? null : tags.stream().map(TagLabel::getTagFQN).collect(Collectors.joining(FIELD_SEPARATOR))); return record; } + + public static List addOwner(List record, EntityReference owner) { + record.add(nullOrEmpty(owner) ? null : owner.getType() + FIELD_SEPARATOR + owner.getName()); + return record; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 637d364bb5f..1e2bdaf8a4f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -144,10 +144,12 @@ public abstract class EntityCsv { /** Owner field is in entityType;entityName format */ public EntityReference getOwner(CSVPrinter printer, CSVRecord record, int fieldNumber) throws IOException { - List list = CsvUtil.fieldToStrings(record.get(fieldNumber)); - if (list == null) { + String owner = record.get(fieldNumber); + if (nullOrEmpty(owner)) { return null; } + + List list = CsvUtil.fieldToStrings(owner); if (list.size() != 2) { importFailure(printer, invalidOwner(fieldNumber), record); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java index e6454b7a2e2..7632de23f3c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java @@ -21,6 +21,7 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.csv.CsvUtil.addEntityReference; import static org.openmetadata.csv.CsvUtil.addEntityReferences; import static org.openmetadata.csv.CsvUtil.addField; +import static org.openmetadata.csv.CsvUtil.addOwner; import static org.openmetadata.csv.CsvUtil.addTagLabels; import com.fasterxml.jackson.core.JsonProcessingException; @@ -39,6 +40,7 @@ import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.api.data.TermReference; import org.openmetadata.schema.entity.data.Glossary; import org.openmetadata.schema.entity.data.GlossaryTerm; +import org.openmetadata.schema.entity.data.GlossaryTerm.Status; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.ProviderType; @@ -122,7 +124,7 @@ public class GlossaryRepository extends EntityRepository { Glossary glossary = getByName(null, name, Fields.EMPTY_FIELDS); // Validate glossary name GlossaryTermRepository repository = (GlossaryTermRepository) Entity.getEntityRepository(Entity.GLOSSARY_TERM); ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parent", name); - List terms = repository.listAll(repository.getFields("reviewers,tags,relatedTerms"), filter); + List terms = repository.listAll(repository.getFields("owner,reviewers,tags,relatedTerms"), filter); terms.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName)); return new GlossaryCsv(glossary, user).exportCsv(terms); } @@ -183,6 +185,17 @@ public class GlossaryRepository extends EntityRepository { if (!processRecord) { return null; } + + // Field 9 - reviewers + glossaryTerm.withReviewers(getEntityReferences(printer, record, 8, Entity.USER)); + if (!processRecord) { + return null; + } + // Field 10 - owner + glossaryTerm.withOwner(getOwner(printer, record, 9)); + + // Field 11 - status + glossaryTerm.withStatus(getTermStatus(printer, record)); return glossaryTerm; } @@ -205,6 +218,19 @@ public class GlossaryRepository extends EntityRepository { return list; } + private Status getTermStatus(CSVPrinter printer, CSVRecord record) throws IOException { + String termStatus = record.get(10); + try { + return nullOrEmpty(termStatus) ? Status.DRAFT : Status.fromValue(termStatus); + } catch (Exception ex) { + // List should have even numbered terms - termName and endPoint + importFailure( + printer, invalidField(10, String.format("Glossary term status %s is invalid", termStatus)), record); + processRecord = false; + return null; + } + } + @Override protected List toRecord(GlossaryTerm entity) { List record = new ArrayList<>(); @@ -216,6 +242,9 @@ public class GlossaryRepository extends EntityRepository { addEntityReferences(record, entity.getRelatedTerms()); addField(record, termReferencesToRecord(entity.getReferences())); addTagLabels(record, entity.getTags()); + addEntityReferences(record, entity.getReviewers()); + addOwner(record, entity.getOwner()); + addField(record, entity.getStatus().value()); return record; } diff --git a/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json b/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json index 1fc31b6d8ea..c9d3713a183 100644 --- a/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json +++ b/openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json @@ -67,6 +67,33 @@ "`PII.Sensitive`", "`PII.Sensitive;PersonalData.Personal`" ] + }, + { + "name": "reviewers", + "required": false, + "description": "User names separated by ';'.", + "examples": [ + "`john;adam`" + ] + }, + { + "name": "owner", + "required": false, + "description": "Owner names separated by ';'. For team owner, include prefix team. For user owner, include prefix user.", + "examples": [ + "`team;marketing`", + "`user;john`" + ] + }, + { + "name": "status", + "required": false, + "description": "Status of the glossary term. Allowed values `Draft`, `Approved`, or `Deprecated`", + "examples": [ + "`Draft`", + "`Approved`", + "`Deprecated`" + ] } ] } \ No newline at end of file diff --git a/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java b/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java index 0741d5424df..13172a778c9 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java @@ -91,9 +91,9 @@ public class EntityCsvTest { int expectedRowsPassed, int expectedRowsFailed) { assertEquals(expectedStatus, importResult.getStatus(), importResult.toString()); - assertEquals(expectedRowsProcessed, importResult.getNumberOfRowsProcessed()); - assertEquals(expectedRowsPassed, importResult.getNumberOfRowsPassed()); - assertEquals(expectedRowsFailed, importResult.getNumberOfRowsFailed()); + assertEquals(expectedRowsProcessed, importResult.getNumberOfRowsProcessed(), importResult.getImportResultsCsv()); + assertEquals(expectedRowsPassed, importResult.getNumberOfRowsPassed(), importResult.getImportResultsCsv()); + assertEquals(expectedRowsFailed, importResult.getNumberOfRowsFailed(), importResult.getImportResultsCsv()); } public static void assertRows(CsvImportResult importResult, String... expectedRows) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index a601e97d72f..eb14bcfd8d5 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -328,7 +328,7 @@ public class GlossaryResourceTest extends EntityResourceTest createRecords = listOf( - ",g1,dsp1,\"dsc1,1\",h1;h2;h3,,term1;http://term1,Tier.Tier1", - ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,Tier.Tier2", - "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,"); + String.format( + ",g1,dsp1,\"dsc1,1\",h1;h2;h3,,term1;http://term1,Tier.Tier1,%s;%s,user;%s,%s", + user1, user2, user1, "Approved"), + String.format( + ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,Tier.Tier2,%s,user;%s,%s", user1, user2, "Draft"), + String.format("importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,%s,team;%s,%s", user1, team1, "Deprecated")); // Update terms with change in description List updateRecords = listOf( - ",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1", - ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,Tier.Tier2", - "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,"); + String.format( + ",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,%s;%s,user;%s,%s", + user1, user2, user1, "Approved"), + String.format( + ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,Tier.Tier2,%s,user;%s,%s", user1, user2, "Draft"), + String.format( + "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,%s,team;%s,%s", user1, team1, "Deprecated")); // Add new row to existing rows - List newRecords = listOf(",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,Tier.Tier3"); + List newRecords = listOf(",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,Tier.Tier3,,,Draft"); testImportExport(glossary.getName(), GlossaryCsv.HEADERS, createRecords, updateRecords, newRecords); }