yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-01 02:56:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #10241 - Include Status, Owner and Reviewer in Bulk Glossary Upload (#10881)

Browse Source
This commit is contained in:
Suresh Srinivas 2023-04-02 15:04:00 -07:00 committed by GitHub
parent 2759a70379
commit cdce2ca08c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 90 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java
View File

@ -135,4 +135,9 @@ public final class CsvUtil {
nullOrEmpty(tags) ? null : tags.stream().map(TagLabel::getTagFQN).collect(Collectors.joining(FIELD_SEPARATOR)));
return record;
}
public static List<String> addOwner(List<String> record, EntityReference owner) {
record.add(nullOrEmpty(owner) ? null : owner.getType() + FIELD_SEPARATOR + owner.getName());
return record;
}
}

6
openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java
View File

@ -144,10 +144,12 @@ public abstract class EntityCsv<T extends EntityInterface> {
/** Owner field is in entityType;entityName format */
public EntityReference getOwner(CSVPrinter printer, CSVRecord record, int fieldNumber) throws IOException {
List<String> list = CsvUtil.fieldToStrings(record.get(fieldNumber));
if (list == null) {
String owner = record.get(fieldNumber);
if (nullOrEmpty(owner)) {
return null;
}
List<String> list = CsvUtil.fieldToStrings(owner);
if (list.size() != 2) {
importFailure(printer, invalidOwner(fieldNumber), record);
}

31
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java
View File

@ -21,6 +21,7 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
import static org.openmetadata.csv.CsvUtil.addEntityReference;
import static org.openmetadata.csv.CsvUtil.addEntityReferences;
import static org.openmetadata.csv.CsvUtil.addField;
import static org.openmetadata.csv.CsvUtil.addOwner;
import static org.openmetadata.csv.CsvUtil.addTagLabels;
import com.fasterxml.jackson.core.JsonProcessingException;
@ -39,6 +40,7 @@ import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.api.data.TermReference;
import org.openmetadata.schema.entity.data.Glossary;
import org.openmetadata.schema.entity.data.GlossaryTerm;
import org.openmetadata.schema.entity.data.GlossaryTerm.Status;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.ProviderType;
@ -122,7 +124,7 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
Glossary glossary = getByName(null, name, Fields.EMPTY_FIELDS); // Validate glossary name
GlossaryTermRepository repository = (GlossaryTermRepository) Entity.getEntityRepository(Entity.GLOSSARY_TERM);
ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parent", name);
List<GlossaryTerm> terms = repository.listAll(repository.getFields("reviewers,tags,relatedTerms"), filter);
List<GlossaryTerm> terms = repository.listAll(repository.getFields("owner,reviewers,tags,relatedTerms"), filter);
terms.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName));
return new GlossaryCsv(glossary, user).exportCsv(terms);
}
@ -183,6 +185,17 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
if (!processRecord) {
return null;
}
// Field 9 - reviewers
glossaryTerm.withReviewers(getEntityReferences(printer, record, 8, Entity.USER));
if (!processRecord) {
return null;
}
// Field 10 - owner
glossaryTerm.withOwner(getOwner(printer, record, 9));
// Field 11 - status
glossaryTerm.withStatus(getTermStatus(printer, record));
return glossaryTerm;
}
@ -205,6 +218,19 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
return list;
}
private Status getTermStatus(CSVPrinter printer, CSVRecord record) throws IOException {
String termStatus = record.get(10);
try {
return nullOrEmpty(termStatus) ? Status.DRAFT : Status.fromValue(termStatus);
} catch (Exception ex) {
// List should have even numbered terms - termName and endPoint
importFailure(
printer, invalidField(10, String.format("Glossary term status %s is invalid", termStatus)), record);
processRecord = false;
return null;
}
}
@Override
protected List<String> toRecord(GlossaryTerm entity) {
List<String> record = new ArrayList<>();
@ -216,6 +242,9 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
addEntityReferences(record, entity.getRelatedTerms());
addField(record, termReferencesToRecord(entity.getReferences()));
addTagLabels(record, entity.getTags());
addEntityReferences(record, entity.getReviewers());
addOwner(record, entity.getOwner());
addField(record, entity.getStatus().value());
return record;
}

27
openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json
View File

@ -67,6 +67,33 @@
"`PII.Sensitive`",
"`PII.Sensitive;PersonalData.Personal`"
]
},
{
"name": "reviewers",
"required": false,
"description": "User names separated by ';'.",
"examples": [
"`john;adam`"
]
},
{
"name": "owner",
"required": false,
"description": "Owner names separated by ';'. For team owner, include prefix team. For user owner, include prefix user.",
"examples": [
"`team;marketing`",
"`user;john`"
]
},
{
"name": "status",
"required": false,
"description": "Status of the glossary term. Allowed values `Draft`, `Approved`, or `Deprecated`",
"examples": [
"`Draft`",
"`Approved`",
"`Deprecated`"
]
}
]
}

6
openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java
View File

@ -91,9 +91,9 @@ public class EntityCsvTest {
int expectedRowsPassed,
int expectedRowsFailed) {
assertEquals(expectedStatus, importResult.getStatus(), importResult.toString());
assertEquals(expectedRowsProcessed, importResult.getNumberOfRowsProcessed());
assertEquals(expectedRowsPassed, importResult.getNumberOfRowsPassed());
assertEquals(expectedRowsFailed, importResult.getNumberOfRowsFailed());
assertEquals(expectedRowsProcessed, importResult.getNumberOfRowsProcessed(), importResult.getImportResultsCsv());
assertEquals(expectedRowsPassed, importResult.getNumberOfRowsPassed(), importResult.getImportResultsCsv());
assertEquals(expectedRowsFailed, importResult.getNumberOfRowsFailed(), importResult.getImportResultsCsv());
}
public static void assertRows(CsvImportResult importResult, String... expectedRows) {

31
openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java
View File

@ -328,7 +328,7 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
// Create glossaryTerm with invalid parent
String resultsHeader = recordToString(EntityCsv.getResultHeaders(GlossaryCsv.HEADERS));
String record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1";
String record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,,,";
String csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null);
CsvImportResult result = importCsv(glossaryName, csv, false);
assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1);
@ -336,7 +336,7 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
assertRows(result, expectedRows);
// Create glossaryTerm with invalid tags field
record = ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tag.invalidTag";
record = ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tag.invalidTag,,,";
csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null);
result = importCsv(glossaryName, csv, false);
assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1);
@ -347,24 +347,35 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
@Test
void testGlossaryImportExport() throws IOException {
Glossary glossary = createEntity(createRequest("importExportTest"), ADMIN_AUTH_HEADERS);
String user1 = USER1.getName();
String user2 = USER2.getName();
String team1 = TEAM1.getName();
// CSV Header "parent" "name" "displayName" "description" "synonyms" "relatedTerms" "references" "tags"
// CSV Header "parent" "name" "displayName" "description" "synonyms" "relatedTerms" "references" "tags",
// "reviewers", "owner", "status"
// Create two records
List<String> createRecords =
listOf(
",g1,dsp1,\"dsc1,1\",h1;h2;h3,,term1;http://term1,Tier.Tier1",
",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,Tier.Tier2",
"importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,");
String.format(
",g1,dsp1,\"dsc1,1\",h1;h2;h3,,term1;http://term1,Tier.Tier1,%s;%s,user;%s,%s",
user1, user2, user1, "Approved"),
String.format(
",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,Tier.Tier2,%s,user;%s,%s", user1, user2, "Draft"),
String.format("importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,%s,team;%s,%s", user1, team1, "Deprecated"));
// Update terms with change in description
List<String> updateRecords =
listOf(
",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1",
",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,Tier.Tier2",
"importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,");
String.format(
",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,%s;%s,user;%s,%s",
user1, user2, user1, "Approved"),
String.format(
",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,Tier.Tier2,%s,user;%s,%s", user1, user2, "Draft"),
String.format(
"importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,%s,team;%s,%s", user1, team1, "Deprecated"));
// Add new row to existing rows
List<String> newRecords = listOf(",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,Tier.Tier3");
List<String> newRecords = listOf(",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,Tier.Tier3,,,Draft");
testImportExport(glossary.getName(), GlossaryCsv.HEADERS, createRecords, updateRecords, newRecords);
}