Fixes #9986 - Add documentation support for the CSV template for imports and exports (#9987)

openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java

@@ -30,15 +30,18 @@ import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import javax.ws.rs.core.Response;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVFormat.Builder;
 import org.apache.commons.csv.CSVPrinter;
 import org.apache.commons.csv.CSVRecord;
+import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.EntityInterface;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.schema.type.TagLabel;
 import org.openmetadata.schema.type.TagLabel.TagSource;
+import org.openmetadata.schema.type.csv.CsvDocumentation;
 import org.openmetadata.schema.type.csv.CsvErrorType;
 import org.openmetadata.schema.type.csv.CsvFile;
 import org.openmetadata.schema.type.csv.CsvHeader;
@@ -46,6 +49,8 @@ import org.openmetadata.schema.type.csv.CsvImportResult;
 import org.openmetadata.schema.type.csv.CsvImportResult.Status;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.jdbi3.EntityRepository;
+import org.openmetadata.service.util.EntityUtil;
+import org.openmetadata.service.util.JsonUtils;
 import org.openmetadata.service.util.RestUtil.PutResponse;
 
 /**
@@ -53,6 +58,7 @@ import org.openmetadata.service.util.RestUtil.PutResponse;
  * provide entity specific processing functionality to export an entity to a CSV record, and import an entity from a CSV
  * record.
  */
+@Slf4j
 public abstract class EntityCsv<T extends EntityInterface> {
   public static final String IMPORT_STATUS_HEADER = "status";
   public static final String IMPORT_STATUS_DETAILS = "details";
@@ -120,6 +126,19 @@ public abstract class EntityCsv<T extends EntityInterface> {
     return CsvUtil.formatCsv(csvFile);
   }
 
+  public static CsvDocumentation getCsvDocumentation(String entityType) {
+    LOG.info("Initializing CSV documentation for entity {}", entityType);
+    String path = String.format(".*json/data/%s/%sCsvDocumentation.json$", entityType, entityType);
+    try {
+      List<String> jsonDataFiles = EntityUtil.getJsonDataResources(path);
+      String json = CommonUtil.getResourceAsStream(EntityRepository.class.getClassLoader(), jsonDataFiles.get(0));
+      return JsonUtils.readValue(json, CsvDocumentation.class);
+    } catch (IOException e) {
+      LOG.error("FATAL - Failed to load CSV documentation for entity {} from the path {}", entityType, path);
+    }
+    return null;
+  }
+
   /** Implement this method to turn an entity into a list of fields */
   protected abstract List<String> toRecord(T entity);
 

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java

@@ -45,6 +45,7 @@ import org.openmetadata.schema.type.ProviderType;
 import org.openmetadata.schema.type.Relationship;
 import org.openmetadata.schema.type.TagLabel;
 import org.openmetadata.schema.type.TagLabel.TagSource;
+import org.openmetadata.schema.type.csv.CsvDocumentation;
 import org.openmetadata.schema.type.csv.CsvHeader;
 import org.openmetadata.schema.type.csv.CsvImportResult;
 import org.openmetadata.service.Entity;
@@ -142,22 +143,12 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
   }
 
   public static class GlossaryCsv extends EntityCsv<GlossaryTerm> {
-    public static final List<CsvHeader> HEADERS = new ArrayList<>();
+    public static final CsvDocumentation DOCUMENTATION = getCsvDocumentation(Entity.GLOSSARY);
+    public static final List<CsvHeader> HEADERS = DOCUMENTATION.getHeaders();
     private final Glossary glossary;
 
-    static {
-      HEADERS.add(new CsvHeader().withName("parent").withRequired(false));
-      HEADERS.add(new CsvHeader().withName("name").withRequired(true));
-      HEADERS.add(new CsvHeader().withName("displayName").withRequired(false));
-      HEADERS.add(new CsvHeader().withName("description").withRequired(true));
-      HEADERS.add(new CsvHeader().withName("synonyms").withRequired(false));
-      HEADERS.add(new CsvHeader().withName("relatedTerms").withRequired(false));
-      HEADERS.add(new CsvHeader().withName("references").withRequired(false));
-      HEADERS.add(new CsvHeader().withName("tags").withRequired(false));
-    }
-
     GlossaryCsv(Glossary glossary, String user) {
-      super(Entity.GLOSSARY_TERM, HEADERS, user);
+      super(Entity.GLOSSARY_TERM, DOCUMENTATION.getHeaders(), user);
       this.glossary = glossary;
     }
 
@@ -184,7 +175,7 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
       }
 
       // Field 7 - TermReferences
-      glossaryTerm.withReferences(getTermReferences(printer, record, 6));
+      glossaryTerm.withReferences(getTermReferences(printer, record));
       if (!processRecord) {
         return null;
       }
@@ -197,16 +188,15 @@ public class GlossaryRepository extends EntityRepository<Glossary> {
       return glossaryTerm;
     }
 
-    private List<TermReference> getTermReferences(CSVPrinter printer, CSVRecord record, int fieldNumber)
+    private List<TermReference> getTermReferences(CSVPrinter printer, CSVRecord record) throws IOException {
-        throws IOException {
+      String termRefs = record.get(6);
-      String termRefs = record.get(fieldNumber);
       if (nullOrEmpty(termRefs)) {
         return null;
       }
       List<String> termRefList = CsvUtil.fieldToStrings(termRefs);
       if (termRefList.size() % 2 != 0) {
         // List should have even numbered terms - termName and endPoint
-        importFailure(printer, invalidField(fieldNumber, "Term references should termName;endpoint"), record);
+        importFailure(printer, invalidField(6, "Term references should termName;endpoint"), record);
         processRecord = false;
         return null;
       }

openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java

@@ -54,11 +54,13 @@ import org.openmetadata.schema.type.csv.CsvImportResult;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.jdbi3.CollectionDAO;
 import org.openmetadata.service.jdbi3.GlossaryRepository;
+import org.openmetadata.service.jdbi3.GlossaryRepository.GlossaryCsv;
 import org.openmetadata.service.jdbi3.ListFilter;
 import org.openmetadata.service.resources.Collection;
 import org.openmetadata.service.resources.EntityResource;
 import org.openmetadata.service.resources.glossary.GlossaryTermResource.GlossaryTermList;
 import org.openmetadata.service.security.Authorizer;
+import org.openmetadata.service.util.JsonUtils;
 import org.openmetadata.service.util.RestUtil;
 import org.openmetadata.service.util.ResultList;
 
@@ -391,6 +393,15 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
     return restoreEntity(uriInfo, securityContext, restore.getId());
   }
 
+  @GET
+  @Path("/documentation/csv")
+  @Valid
+  @Operation(operationId = "getCsvDocumentation", summary = "Get CSV documentation", tags = "glossaries")
+  public String getCsvDocumentation(@Context SecurityContext securityContext, @PathParam("name") String name)
+      throws IOException {
+    return JsonUtils.pojoToJson(GlossaryCsv.DOCUMENTATION);
+  }
+
   @GET
   @Path("/name/{name}/export")
   @Produces(MediaType.TEXT_PLAIN)
@@ -402,9 +413,8 @@ public class GlossaryResource extends EntityResource<Glossary, GlossaryRepositor
       responses = {
         @ApiResponse(
             responseCode = "200",
-            description = "List of glossary terms",
+            description = "CSV file",
-            content =
+            content = @Content(mediaType = "application/json", schema = @Schema(implementation = String.class)))
-                @Content(mediaType = "application/json", schema = @Schema(implementation = GlossaryTermList.class)))
       })
   public String exportCsv(@Context SecurityContext securityContext, @PathParam("name") String name) throws IOException {
     return super.exportCsvInternal(securityContext, name);

openmetadata-service/src/main/resources/json/data/glossary/glossaryCsvDocumentation.json

@@ -0,0 +1,70 @@
+{
+  "summary": "Glossary CSV file is used for importing and exporting glossary terms from and to an **existing** glossary.",
+  "headers": [
+    {
+      "name": "parent",
+      "required": false,
+      "description": "Fully qualified name of the parent glossary term. If the glossary term being created is at the root of the glossary without any parent term, leave this as empty.",
+      "examples": [
+        "`\"\"` or empty, if the term is at the root of the glossary.",
+        "`Business terms.Customer` as parent to create a term `CustomerId` under it."
+      ]
+    },
+    {
+      "name": "name",
+      "required": true,
+      "description": "The name of the glossary term being created.",
+      "examples": [
+        "`CustomerId`, `Customer name`"
+      ]
+    },
+    {
+      "name": "displayName",
+      "required": false,
+      "description": "Display name for the term.",
+      "examples": [
+        "`Customer Identifier`, `Customer name`"
+      ]
+    },
+    {
+      "name": "description",
+      "required": false,
+      "description": "Description for the glossary term in markdown format.",
+      "examples": [
+        "`Customer Identifier` as defined by the **Legal Team**."
+      ]
+    },
+    {
+      "name": "synonyms",
+      "required": false,
+      "description": "Synonyms for the glossary term",
+      "examples": [
+        "`Customer Identifier`, `cid`, `customer_id`"
+      ]
+    },
+    {
+      "name": "relatedTerms",
+      "required": false,
+      "description": "List of related glossary term **fully qualified names** separated by `;`.",
+      "examples": [
+        "`Business terms.Client Identifier`, `Support.Subscriber Id`"
+      ]
+    },
+    {
+      "name": "references",
+      "required": false,
+      "description": "External glossary references for the glossary term in the format `name;URL endPoint`.",
+      "examples": [
+        "`customer;http://domain.com/glossaries/customer`"
+      ]
+    },
+    {
+      "name": "tags",
+      "required": false,
+      "description": "Classification tags associated with the glossary term. These tags are automatically applied along with the glossary term, when it is used to label an entity.",
+      "examples": [
+        "`customer;http://domain.com/glossaries/customer`"
+      ]
+    }
+  ]
+}

openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java

@@ -90,7 +90,7 @@ public class EntityCsvTest {
       int expectedRowsProcessed,
       int expectedRowsPassed,
       int expectedRowsFailed) {
-    assertEquals(expectedStatus, importResult.getStatus());
+    assertEquals(expectedStatus, importResult.getStatus(), importResult.getImportResultsCsv());
     assertEquals(expectedRowsProcessed, importResult.getNumberOfRowsProcessed());
     assertEquals(expectedRowsPassed, importResult.getNumberOfRowsPassed());
     assertEquals(expectedRowsFailed, importResult.getNumberOfRowsFailed());

openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java

@@ -66,6 +66,7 @@ import org.openmetadata.schema.type.ColumnDataType;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.ProviderType;
 import org.openmetadata.schema.type.TagLabel;
+import org.openmetadata.schema.type.csv.CsvDocumentation;
 import org.openmetadata.schema.type.csv.CsvHeader;
 import org.openmetadata.schema.type.csv.CsvImportResult;
 import org.openmetadata.service.Entity;
@@ -306,6 +307,11 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
     }
   }
 
+  @Test
+  void testGlossaryCsvDocumentation() throws HttpResponseException {
+    assertEquals(GlossaryCsv.DOCUMENTATION, getCsvDocumentation());
+  }
+
   @Test
   void testGlossaryImportInvalidCsv() throws IOException {
     String glossaryName = "invalidCsv";
@@ -381,6 +387,11 @@ public class GlossaryResourceTest extends EntityResourceTest<Glossary, CreateGlo
     assertEquals(csv, exportedCsv);
   }
 
+  private CsvDocumentation getCsvDocumentation() throws HttpResponseException {
+    WebTarget target = getCollection().path("/documentation/csv");
+    return TestUtils.get(target, CsvDocumentation.class, ADMIN_AUTH_HEADERS);
+  }
+
   private CsvImportResult importCsv(String glossaryName, String csv, boolean dryRun) throws HttpResponseException {
     WebTarget target = getResourceByName(glossaryName).path("/import");
     target = !dryRun ? target.queryParam("dryRun", false) : target;

openmetadata-spec/src/main/resources/json/schema/type/csvDocumentation.json

@@ -0,0 +1,24 @@
+{
+  "$id": "https://open-metadata.org/schema/type/csvDocumentation.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "csvDocumentation",
+  "description": "Documentation for CSV file that describes headers and example values.",
+  "javaType": "org.openmetadata.schema.type.csv.CsvDocumentation",
+  "type": "object",
+  "properties": {
+    "summary" : {
+      "description": "Summary documentation for CSV file.",
+      "type" : "string"
+
+    },
+    "headers": {
+      "description": "Documentation for CSV file header",
+      "type": "array",
+      "items": {
+        "$ref": "csvFile.json#/definitions/csvHeader"
+      }
+    }
+  },
+  "required": ["summary", "headers"],
+  "additionalProperties": false
+}

openmetadata-spec/src/main/resources/json/schema/type/csvFile.json

@@ -16,9 +16,21 @@
         "required": {
           "type": "boolean",
           "default": false
+        },
+        "description" : {
+          "description": "Description of the header field for documentation purposes.",
+          "$ref" : "basic.json#/definitions/markdown"
+        },
+        "examples" : {
+          "description": "Example values for the field",
+          "type" : "array",
+          "items": {
+            "type": "string"
+          }
         }
       },
-      "additionalProperties": false
+      "additionalProperties": false,
+      "required": ["name", "description", "examples"]
     },
     "csvRecord": {
       "javaType": "org.openmetadata.schema.type.csv.CsvRecord",