yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-25 17:04:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #9986 - Add documentation support for the CSV template for imports and exports (#9987)

Browse Source
This commit is contained in:
Suresh Srinivas 2023-01-29 13:47:02 -08:00 committed by GitHub
parent 95539b7008
commit d81187817a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 159 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java
View File

@ -30,15 +30,18 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import javax.ws.rs.core.Response;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVFormat.Builder;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.CSVRecord;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.TagLabel.TagSource;
import org.openmetadata.schema.type.csv.CsvDocumentation;
import org.openmetadata.schema.type.csv.CsvErrorType;
import org.openmetadata.schema.type.csv.CsvFile;
import org.openmetadata.schema.type.csv.CsvHeader;
@ -46,6 +49,8 @@ import org.openmetadata.schema.type.csv.CsvImportResult;
import org.openmetadata.schema.type.csv.CsvImportResult.Status;
import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.EntityRepository;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.RestUtil.PutResponse;
/**
@ -53,6 +58,7 @@ import org.openmetadata.service.util.RestUtil.PutResponse;
* provide entity specific processing functionality to export an entity to a CSV record, and import an entity from a CSV
* record.
*/
@Slf4j
public abstract class EntityCsv<T extends EntityInterface> {
public static final String IMPORT_STATUS_HEADER = "status";
public static final String IMPORT_STATUS_DETAILS = "details";
@ -120,6 +126,19 @@ public abstract class EntityCsv<T extends EntityInterface> {
return CsvUtil.formatCsv(csvFile);
}
public static CsvDocumentation getCsvDocumentation(String entityType) {
LOG.info("Initializing CSV documentation for entity {}", entityType);
String path = String.format(".*json/data/%s/%sCsvDocumentation.json$", entityType, entityType);
try {
List<String> jsonDataFiles = EntityUtil.getJsonDataResources(path);
String json = CommonUtil.getResourceAsStream(EntityRepository.class.getClassLoader(), jsonDataFiles.get(0));
return JsonUtils.readValue(json, CsvDocumentation.class);
} catch (IOException e) {
LOG.error("FATAL - Failed to load CSV documentation for entity {} from the path {}", entityType, path);
}
return null;
}
/** Implement this method to turn an entity into a list of fields */
protected abstract List<String> toRecord(T entity);

26
openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java
View File

@ -45,6 +45,7 @@ import org.openmetadata.schema.type.ProviderType;
import org.openmetadata.schema.type.Relationship;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.TagLabel.TagSource;
import org.openmetadata.schema.type.csv.CsvDocumentation;
import org.openmetadata.schema.type.csv.CsvHeader;
import org.openmetadata.schema.type.csv.CsvImportResult;
import org.openmetadata.service.Entity;
@ -142,22 +143,12 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
}
public static class GlossaryCsv extends EntityCsv<GlossaryTerm> {
public static final List<CsvHeader> HEADERS = new ArrayList<>();
public static final CsvDocumentation DOCUMENTATION = getCsvDocumentation(Entity.GLOSSARY);
public static final List<CsvHeader> HEADERS = DOCUMENTATION.getHeaders();
private final Glossary glossary;
static {
HEADERS.add(new CsvHeader().withName("parent").withRequired(false));
HEADERS.add(new CsvHeader().withName("name").withRequired(true));
HEADERS.add(new CsvHeader().withName("displayName").withRequired(false));
HEADERS.add(new CsvHeader().withName("description").withRequired(true));
HEADERS.add(new CsvHeader().withName("synonyms").withRequired(false));
HEADERS.add(new CsvHeader().withName("relatedTerms").withRequired(false));
HEADERS.add(new CsvHeader().withName("references").withRequired(false));
HEADERS.add(new CsvHeader().withName("tags").withRequired(false));
}
GlossaryCsv(Glossary glossary, String user) {
super(Entity.GLOSSARY_TERM, HEADERS, user);
super(Entity.GLOSSARY_TERM, DOCUMENTATION.getHeaders(), user);
this.glossary = glossary;
}
@ -184,7 +175,7 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
}
// Field 7 - TermReferences
glossaryTerm.withReferences(getTermReferences(printer, record, 6));
glossaryTerm.withReferences(getTermReferences(printer, record));
if (!processRecord) {
return null;
}
@ -197,16 +188,15 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
return glossaryTerm;
}
private List<TermReference> getTermReferences(CSVPrinter printer, CSVRecord record, int fieldNumber)
throws IOException {
String termRefs = record.get(fieldNumber);
private List<TermReference> getTermReferences(CSVPrinter printer, CSVRecord record) throws IOException {
String termRefs = record.get(6);
if (nullOrEmpty(termRefs)) {
return null;
}
List<String> termRefList = CsvUtil.fieldToStrings(termRefs);
if (termRefList.size() % 2 != 0) {
// List should have even numbered terms - termName and endPoint
importFailure(printer, invalidField(fieldNumber, "Term references should termName;endpoint"), record);
importFailure(printer, invalidField(6, "Term references should termName;endpoint"), record);
processRecord = false;
return null;
}

16
openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java
View File

@ -54,11 +54,13 @@ import org.openmetadata.schema.type.csv.CsvImportResult;
import org.openmetadata.service.Entity;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.GlossaryRepository;
import org.openmetadata.service.jdbi3.GlossaryRepository.GlossaryCsv;
import org.openmetadata.service.jdbi3.ListFilter;
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.resources.glossary.GlossaryTermResource.GlossaryTermList;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.RestUtil;
import org.openmetadata.service.util.ResultList;
@ -391,6 +393,15 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
return restoreEntity(uriInfo, securityContext, restore.getId());
}
@GET
@Path("/documentation/csv")
@Valid
@Operation(operationId = "getCsvDocumentation", summary = "Get CSV documentation", tags = "glossaries")
public String getCsvDocumentation(@Context SecurityContext securityContext, @PathParam("name") String name)
throws IOException {
return JsonUtils.pojoToJson(GlossaryCsv.DOCUMENTATION);
}
@GET
@Path("/name/{name}/export")
@Produces(MediaType.TEXT_PLAIN)
@ -402,9 +413,8 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
responses = {
@ApiResponse(
responseCode = "200",
description = "List of glossary terms",
content =
@Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryTermList.class)))
description = "CSV file",
content = @Content(mediaType = "application/json", schema = @Schema(implementation = String.class)))
})
public String exportCsv(@Context SecurityContext securityContext, @PathParam("name") String name) throws IOException {
return super.exportCsvInternal(securityContext, name);

70
openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json Normal file
View File

@ -0,0 +1,70 @@
{
"summary": "Glossary CSV file is used for importing and exporting glossary terms from and to an **existing** glossary.",
"headers": [
{
"name": "parent",
"required": false,
"description": "Fully qualified name of the parent glossary term. If the glossary term being created is at the root of the glossary without any parent term, leave this as empty.",
"examples": [
"`\"\"` or empty, if the term is at the root of the glossary.",
"`Business terms.Customer` as parent to create a term `CustomerId` under it."
]
},
{
"name": "name",
"required": true,
"description": "The name of the glossary term being created.",
"examples": [
"`CustomerId`, `Customer name`"
]
},
{
"name": "displayName",
"required": false,
"description": "Display name for the term.",
"examples": [
"`Customer Identifier`, `Customer name`"
]
},
{
"name": "description",
"required": false,
"description": "Description for the glossary term in markdown format.",
"examples": [
"`Customer Identifier` as defined by the **Legal Team**."
]
},
{
"name": "synonyms",
"required": false,
"description": "Synonyms for the glossary term",
"examples": [
"`Customer Identifier`, `cid`, `customer_id`"
]
},
{
"name": "relatedTerms",
"required": false,
"description": "List of related glossary term **fully qualified names** separated by `;`.",
"examples": [
"`Business terms.Client Identifier`, `Support.Subscriber Id`"
]
},
{
"name": "references",
"required": false,
"description": "External glossary references for the glossary term in the format `name;URL endPoint`.",
"examples": [
"`customer;http://domain.com/glossaries/customer`"
]
},
{
"name": "tags",
"required": false,
"description": "Classification tags associated with the glossary term. These tags are automatically applied along with the glossary term, when it is used to label an entity.",
"examples": [
"`customer;http://domain.com/glossaries/customer`"
]
}
]
}

2
openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java
View File

@ -90,7 +90,7 @@ public class EntityCsvTest {
int expectedRowsProcessed,
int expectedRowsPassed,
int expectedRowsFailed) {
assertEquals(expectedStatus, importResult.getStatus());
assertEquals(expectedStatus, importResult.getStatus(), importResult.getImportResultsCsv());
assertEquals(expectedRowsProcessed, importResult.getNumberOfRowsProcessed());
assertEquals(expectedRowsPassed, importResult.getNumberOfRowsPassed());
assertEquals(expectedRowsFailed, importResult.getNumberOfRowsFailed());

11
openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java
View File

@ -66,6 +66,7 @@ import org.openmetadata.schema.type.ColumnDataType;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.ProviderType;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.csv.CsvDocumentation;
import org.openmetadata.schema.type.csv.CsvHeader;
import org.openmetadata.schema.type.csv.CsvImportResult;
import org.openmetadata.service.Entity;
@ -306,6 +307,11 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
}
}
@Test
void testGlossaryCsvDocumentation() throws HttpResponseException {
assertEquals(GlossaryCsv.DOCUMENTATION, getCsvDocumentation());
}
@Test
void testGlossaryImportInvalidCsv() throws IOException {
String glossaryName = "invalidCsv";
@ -381,6 +387,11 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
assertEquals(csv, exportedCsv);
}
private CsvDocumentation getCsvDocumentation() throws HttpResponseException {
WebTarget target = getCollection().path("/documentation/csv");
return TestUtils.get(target, CsvDocumentation.class, ADMIN_AUTH_HEADERS);
}
private CsvImportResult importCsv(String glossaryName, String csv, boolean dryRun) throws HttpResponseException {
WebTarget target = getResourceByName(glossaryName).path("/import");
target = !dryRun ? target.queryParam("dryRun", false) : target;

24
openmetadata-spec/src/main/resources/json/schema/type/csvDocumentation.json Normal file
View File

@ -0,0 +1,24 @@
{
"$id": "https://open-metadata.org/schema/type/csvDocumentation.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "csvDocumentation",
"description": "Documentation for CSV file that describes headers and example values.",
"javaType": "org.openmetadata.schema.type.csv.CsvDocumentation",
"type": "object",
"properties": {
"summary" : {
"description": "Summary documentation for CSV file.",
"type" : "string"
},
"headers": {
"description": "Documentation for CSV file header",
"type": "array",
"items": {
"$ref": "csvFile.json#/definitions/csvHeader"
}
}
},
"required": ["summary", "headers"],
"additionalProperties": false
}

14
openmetadata-spec/src/main/resources/json/schema/type/csvFile.json
View File

@ -16,9 +16,21 @@
"required": {
"type": "boolean",
"default": false
},
"description" : {
"description": "Description of the header field for documentation purposes.",
"$ref" : "basic.json#/definitions/markdown"
},
"examples" : {
"description": "Example values for the field",
"type" : "array",
"items": {
"type": "string"
}
}
},
"additionalProperties": false
"additionalProperties": false,
"required": ["name", "description", "examples"]
},
"csvRecord": {
"javaType": "org.openmetadata.schema.type.csv.CsvRecord",