diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml index 09a9f6c1dbe..31ca8716cf2 100644 --- a/openmetadata-service/pom.xml +++ b/openmetadata-service/pom.xml @@ -410,6 +410,11 @@ diff-match-patch ${diffMatch.version} + + org.apache.commons + commons-csv + 1.9.0 + diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java new file mode 100644 index 00000000000..c26bb3f5c5b --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java @@ -0,0 +1,129 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.csv; + +import static org.openmetadata.common.utils.CommonUtil.listOf; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVFormat.Builder; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.CSVRecord; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.schema.type.csv.CsvFile; +import org.openmetadata.schema.type.csv.CsvHeader; + +public final class CsvUtil { + public static String SEPARATOR = ","; + public static String FIELD_SEPARATOR = ";"; + public static String LINE_SEPARATOR = "\r\n"; + + private CsvUtil() { + // Utility class hides the constructor + } + + public static String formatCsv(CsvFile csvFile) throws IOException { + // CSV file is generated by the backend and the data exported is expected to be correct. Hence, no validation + StringWriter writer = new StringWriter(); + List headers = getHeaders(csvFile.getHeaders()); + CSVFormat csvFormat = Builder.create(CSVFormat.DEFAULT).setHeader(headers.toArray(new String[0])).build(); + try (CSVPrinter printer = new CSVPrinter(writer, csvFormat)) { + for (List record : listOrEmpty(csvFile.getRecords())) { + printer.printRecord(record); + } + } + return writer.toString(); + } + + /** Get headers from CsvHeaders */ + public static List getHeaders(List csvHeaders) { + List headers = new ArrayList<>(); + for (CsvHeader header : csvHeaders) { + String headerString = header.getRequired() ? String.format("%s*", header.getName()) : header.getName(); + headers.add(quoteField(headerString)); + } + return headers; + } + + public static String recordToString(CSVRecord record) { + return recordToString(record.toList()); + } + + public static String recordToString(List fields) { + return String.join(SEPARATOR, fields); + } + + public static String recordToString(String[] fields) { + return String.join(SEPARATOR, fields); + } + + public static List fieldToStrings(String field) { + // Split a field that contains multiple strings separated by FIELD_SEPARATOR + return field == null ? null : listOf(field.split(FIELD_SEPARATOR)); + } + + public static String quote(String field) { + return String.format("\"%s\"", field); + } + + /** Quote a CSV field that has SEPARATOR with " " */ + public static String quoteField(String field) { + return field == null ? "" : field.contains(SEPARATOR) ? quote(field) : field; + } + + /** Quote a CSV field made of multiple strings that has SEPARATOR or FIELD_SEPARATOR with " " */ + public static String quoteField(List field) { + return nullOrEmpty(field) + ? "" + : field.stream() + .map(str -> str.contains(SEPARATOR) || str.contains(FIELD_SEPARATOR) ? quote(str) : str) + .collect(Collectors.joining(FIELD_SEPARATOR)); + } + + public static List addField(List record, String field) { + record.add(quoteField(field)); + return record; + } + + public static List addFieldList(List record, List field) { + record.add(quoteField(field)); + return record; + } + + public static List addEntityReferences(List record, List refs) { + record.add( + nullOrEmpty(refs) + ? null + : refs.stream().map(EntityReference::getFullyQualifiedName).collect(Collectors.joining(FIELD_SEPARATOR))); + return record; + } + + public static List addEntityReference(List record, EntityReference ref) { + record.add(nullOrEmpty(ref) ? null : quoteField(ref.getFullyQualifiedName())); + return record; + } + + public static List addTagLabels(List record, List tags) { + record.add( + nullOrEmpty(tags) ? null : tags.stream().map(TagLabel::getTagFQN).collect(Collectors.joining(FIELD_SEPARATOR))); + return record; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java new file mode 100644 index 00000000000..bb48917be28 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -0,0 +1,345 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.csv; + +import static org.openmetadata.common.utils.CommonUtil.listOf; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.csv.CsvUtil.FIELD_SEPARATOR; +import static org.openmetadata.csv.CsvUtil.recordToString; + +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import javax.ws.rs.core.Response; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVFormat.Builder; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.CSVRecord; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.schema.type.TagLabel.TagSource; +import org.openmetadata.schema.type.csv.CsvErrorType; +import org.openmetadata.schema.type.csv.CsvFile; +import org.openmetadata.schema.type.csv.CsvHeader; +import org.openmetadata.schema.type.csv.CsvImportResult; +import org.openmetadata.schema.type.csv.CsvImportResult.Status; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.util.RestUtil.PutResponse; + +/** + * EntityCsv provides export and import capabilities for an entity. Each entity must implement the abstract methods to + * provide entity specific processing functionality to export an entity to a CSV record, and import an entity from a CSV + * record. + */ +public abstract class EntityCsv { + public static final String IMPORT_STATUS_HEADER = "status"; + public static final String IMPORT_STATUS_DETAILS = "details"; + public static final String IMPORT_STATUS_SUCCESS = "success"; + public static final String IMPORT_STATUS_FAILED = "failure"; + private final String entityType; + private final List csvHeaders; + private final CsvImportResult importResult = new CsvImportResult(); + protected boolean processRecord; // When set to false record processing is discontinued + public static final String ENTITY_CREATED = "Entity created"; + public static final String ENTITY_UPDATED = "Entity updated"; + private final Map dryRunCreatedEntities = new HashMap<>(); + private final String user; + + protected EntityCsv(String entityType, List csvHeaders, String user) { + this.entityType = entityType; + this.csvHeaders = csvHeaders; + this.user = user; + } + + // Import entities from the CSV file + public final CsvImportResult importCsv(String csv, boolean dryRun) throws IOException { + importResult.withDryRun(dryRun); + StringWriter writer = new StringWriter(); + CSVPrinter resultsPrinter = getResultsCsv(csvHeaders, writer); + if (resultsPrinter == null) { + return importResult; + } + + // Parse CSV + Iterator records = parse(csv); + if (records == null) { + return importResult; // Error during parsing + } + + // Validate headers + List expectedHeaders = CsvUtil.getHeaders(csvHeaders); + if (!validateHeaders(expectedHeaders, records.next())) { + return importResult; + } + importResult.withNumberOfRowsPassed(importResult.getNumberOfRowsPassed() + 1); + + // Validate and load each record + while (records.hasNext()) { + CSVRecord record = records.next(); + processRecord(resultsPrinter, expectedHeaders, record); + } + + // Finally, create the entities parsed from the record + setFinalStatus(); + importResult.withImportResultsCsv(writer.toString()); + return importResult; + } + + /** Implement this method to validate each record */ + protected abstract T toEntity(CSVPrinter resultsPrinter, CSVRecord record) throws IOException; + + public final String exportCsv(List entities) throws IOException { + CsvFile csvFile = new CsvFile().withHeaders(csvHeaders); + List> records = new ArrayList<>(); + for (T entity : entities) { + records.add(toRecord(entity)); + } + csvFile.withRecords(records); + return CsvUtil.formatCsv(csvFile); + } + + /** Implement this method to turn an entity into a list of fields */ + protected abstract List toRecord(T entity); + + protected final EntityReference getEntityReference( + CSVPrinter printer, CSVRecord record, int fieldNumber, String entityType) throws IOException { + String fqn = record.get(fieldNumber); + return getEntityReference(printer, record, fieldNumber, entityType, fqn); + } + + private EntityInterface getEntity(String entityType, String fqn) { + EntityInterface entity = entityType.equals(this.entityType) ? dryRunCreatedEntities.get(fqn) : null; + if (entity == null) { + EntityRepository entityRepository = Entity.getEntityRepository(entityType); + entity = entityRepository.findByNameOrNull(fqn, "", Include.NON_DELETED); + } + return entity; + } + + protected final EntityReference getEntityReference( + CSVPrinter printer, CSVRecord record, int fieldNumber, String entityType, String fqn) throws IOException { + if (nullOrEmpty(fqn)) { + return null; + } + EntityInterface entity = getEntity(entityType, fqn); + if (entity == null) { + importFailure(printer, entityNotFound(fieldNumber, fqn), record); + processRecord = false; + return null; + } + return entity.getEntityReference(); + } + + protected final List getEntityReferences( + CSVPrinter printer, CSVRecord record, int fieldNumber, String entityType) throws IOException { + String fqns = record.get(fieldNumber); + if (nullOrEmpty(fqns)) { + return null; + } + List fqnList = listOrEmpty(CsvUtil.fieldToStrings(fqns)); + List refs = new ArrayList<>(); + for (String fqn : fqnList) { + EntityReference ref = getEntityReference(printer, record, fieldNumber, entityType, fqn); + if (!processRecord) { + return null; + } + if (ref != null) { + refs.add(ref); + } + } + return refs.isEmpty() ? null : refs; + } + + protected final List getTagLabels(CSVPrinter printer, CSVRecord record, int fieldNumber) + throws IOException { + List refs = getEntityReferences(printer, record, fieldNumber, Entity.TAG); + if (!processRecord || nullOrEmpty(refs)) { + return null; + } + List tagLabels = new ArrayList<>(); + for (EntityReference ref : refs) { + tagLabels.add(new TagLabel().withSource(TagSource.TAG).withTagFQN(ref.getFullyQualifiedName())); + } + return tagLabels; + } + + public static String[] getResultHeaders(List csvHeaders) { + List importResultsCsvHeader = listOf(IMPORT_STATUS_HEADER, IMPORT_STATUS_DETAILS); + importResultsCsvHeader.addAll(CsvUtil.getHeaders(csvHeaders)); + return importResultsCsvHeader.toArray(new String[0]); + } + + // Create a CSVPrinter to capture the import results + private CSVPrinter getResultsCsv(List csvHeaders, StringWriter writer) { + CSVFormat format = Builder.create(CSVFormat.DEFAULT).setHeader(getResultHeaders(csvHeaders)).build(); + try { + return new CSVPrinter(writer, format); + } catch (IOException e) { + documentFailure(failed(e.getMessage(), CsvErrorType.UNKNOWN)); + } + return null; + } + + private Iterator parse(String csv) { + Reader in = new StringReader(csv); + try { + return CSVFormat.DEFAULT.parse(in).iterator(); + } catch (IOException e) { + documentFailure(failed(e.getMessage(), CsvErrorType.PARSER_FAILURE)); + } + return null; + } + + private boolean validateHeaders(List expectedHeaders, CSVRecord record) { + importResult.withNumberOfRowsProcessed((int) record.getRecordNumber()); + if (expectedHeaders.equals(record.toList())) { + return true; + } + importResult.withNumberOfRowsFailed(1); + documentFailure(invalidHeader(recordToString(expectedHeaders), recordToString(record))); + return false; + } + + private void processRecord(CSVPrinter resultsPrinter, List expectedHeader, CSVRecord record) + throws IOException { + processRecord = true; + // Every row must have total fields corresponding to the number of headers + if (csvHeaders.size() != record.size()) { + importFailure(resultsPrinter, invalidFieldCount(expectedHeader.size(), record.size()), record); + return; + } + + // Check if required values are present + List errors = new ArrayList<>(); + for (int i = 0; i < csvHeaders.size(); i++) { + String field = record.get(i); + boolean fieldRequired = Boolean.TRUE.equals(csvHeaders.get(i).getRequired()); + if (fieldRequired && nullOrEmpty(field)) { + errors.add(fieldRequired(i)); + } + } + + if (!errors.isEmpty()) { + importFailure(resultsPrinter, String.join(FIELD_SEPARATOR, errors), record); + return; + } + + // Finally, convert record into entity for importing + T entity = toEntity(resultsPrinter, record); + if (entity != null) { + // Finally, create entities + createEntity(resultsPrinter, record, entity); + } + } + + private void createEntity(CSVPrinter resultsPrinter, CSVRecord record, T entity) throws IOException { + entity.setId(UUID.randomUUID()); + entity.setUpdatedBy(user); + entity.setUpdatedAt(System.currentTimeMillis()); + EntityRepository repository = Entity.getEntityRepository(entityType); + Response.Status responseStatus; + if (!importResult.getDryRun()) { + try { + repository.prepareInternal(entity); + PutResponse response = repository.createOrUpdate(null, entity); + responseStatus = response.getStatus(); + } catch (Exception ex) { + importFailure(resultsPrinter, ex.getMessage(), record); + return; + } + } else { + repository.setFullyQualifiedName(entity); + responseStatus = + repository.findByNameOrNull(entity.getFullyQualifiedName(), "", Include.NON_DELETED) == null + ? Response.Status.CREATED + : Response.Status.OK; + // Track the dryRun created entities, as they may be referred by other entities being created during import + dryRunCreatedEntities.put(entity.getFullyQualifiedName(), entity); + } + + if (Response.Status.CREATED.equals(responseStatus)) { + importSuccess(resultsPrinter, record, ENTITY_CREATED); + } else { + importSuccess(resultsPrinter, record, ENTITY_UPDATED); + } + } + + public String failed(String exception, CsvErrorType errorType) { + return String.format("#%s: Failed to parse the CSV filed - reason %s", errorType, exception); + } + + public static String invalidHeader(String expected, String actual) { + return String.format("#%s: Headers [%s] doesn't match [%s]", CsvErrorType.INVALID_HEADER, actual, expected); + } + + public static String invalidFieldCount(int expectedFieldCount, int actualFieldCount) { + return String.format( + "#%s: Field count %d does not match the expected field count of %d", + CsvErrorType.INVALID_FIELD_COUNT, actualFieldCount, expectedFieldCount); + } + + public static String fieldRequired(int field) { + return String.format("#%s: Field %d is required", CsvErrorType.FIELD_REQUIRED, field + 1); + } + + public static String invalidField(int field, String error) { + return String.format("#%s: Field %d error - %s", CsvErrorType.INVALID_FIELD, field + 1, error); + } + + public static String entityNotFound(int field, String fqn) { + String error = String.format("Entity %s not found", fqn); + return String.format("#%s: Field %d error - %s", CsvErrorType.INVALID_FIELD, field + 1, error); + } + + private void documentFailure(String error) { + importResult.withStatus(Status.ABORTED); + importResult.withAbortReason(error); + } + + private void importSuccess(CSVPrinter printer, CSVRecord inputRecord, String successDetails) throws IOException { + List record = listOf(IMPORT_STATUS_SUCCESS, successDetails); + record.addAll(inputRecord.toList()); + printer.printRecord(record); + importResult.withNumberOfRowsProcessed((int) inputRecord.getRecordNumber()); + importResult.withNumberOfRowsPassed(importResult.getNumberOfRowsPassed() + 1); + } + + protected void importFailure(CSVPrinter printer, String failedReason, CSVRecord inputRecord) throws IOException { + List record = listOf(IMPORT_STATUS_FAILED, failedReason); + record.addAll(inputRecord.toList()); + printer.printRecord(record); + importResult.withNumberOfRowsProcessed((int) inputRecord.getRecordNumber()); + importResult.withNumberOfRowsFailed(importResult.getNumberOfRowsFailed() + 1); + processRecord = false; + } + + private void setFinalStatus() { + Status status = + importResult.getNumberOfRowsPassed().equals(importResult.getNumberOfRowsProcessed()) + ? Status.SUCCESS + : importResult.getNumberOfRowsPassed() > 1 ? Status.PARTIAL_SUCCESS : Status.FAILURE; + importResult.setStatus(status); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java index 8429333f0ad..9c56fe8daf0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java @@ -197,4 +197,8 @@ public final class CatalogExceptionMessage { "Tag labels %s and %s are mutually exclusive and can't be assigned together", tag1.getTagFQN(), tag2.getTagFQN()); } + + public static String csvNotSupported(String entityType) { + return String.format("Upload/download CSV for bulk operations is not supported for entity [%s]", entityType); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 3f745f1fb36..21d4e983bb7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -27,6 +27,7 @@ import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; import static org.openmetadata.service.Entity.FIELD_OWNER; import static org.openmetadata.service.Entity.FIELD_TAGS; import static org.openmetadata.service.Entity.getEntityFields; +import static org.openmetadata.service.exception.CatalogExceptionMessage.csvNotSupported; import static org.openmetadata.service.util.EntityUtil.compareTagLabel; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; import static org.openmetadata.service.util.EntityUtil.fieldAdded; @@ -79,6 +80,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.ProviderType; import org.openmetadata.schema.type.Relationship; import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.schema.type.csv.CsvImportResult; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.TypeRegistry; @@ -326,9 +328,25 @@ public abstract class EntityRepository { } @Transaction - public final T findByNameOrNull(String fqn, String fields, Include include) throws IOException { + public final T findByNameOrNull(String fqn, String fields, Include include) { String json = dao.findJsonByFqn(fqn, include); - return json == null ? null : setFieldsInternal(JsonUtils.readValue(json, entityClass), getFields(fields)); + try { + return json == null ? null : setFieldsInternal(JsonUtils.readValue(json, entityClass), getFields(fields)); + } catch (IOException e) { + return null; + } + } + + @Transaction + public final List listAll(Fields fields, ListFilter filter) throws IOException { + // forward scrolling, if after == null then first page is being asked + List jsons = dao.listAfter(filter, Integer.MAX_VALUE, ""); + List entities = new ArrayList<>(); + for (String json : jsons) { + T entity = setFieldsInternal(JsonUtils.readValue(json, entityClass), fields); + entities.add(entity); + } + return entities; } @Transaction @@ -1162,6 +1180,16 @@ public abstract class EntityRepository { return Entity.getEntityReferenceById(owner.getType(), owner.getId(), ALL); } + /** Override this method to support downloading CSV functionality */ + public String exportToCsv(String name, String user) throws IOException { + throw new IllegalArgumentException(csvNotSupported(entityType)); + } + + /** Load CSV provided for bulk upload */ + public CsvImportResult importFromCsv(String name, String csv, boolean dryRun, String user) throws IOException { + throw new IllegalArgumentException(csvNotSupported(entityType)); + } + public enum Operation { PUT, PATCH, @@ -1541,8 +1569,6 @@ public abstract class EntityRepository { String toEntityType, UUID toId) throws JsonProcessingException { - List added = new ArrayList<>(); - List deleted = new ArrayList<>(); if (!recordChange(field, originFromRef, updatedFromRef, true, entityReferenceMatch)) { return; // No changes between original and updated. } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java index b8f19423641..0d473635ede 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java @@ -17,17 +17,36 @@ package org.openmetadata.service.jdbi3; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.csv.CsvUtil.addEntityReference; +import static org.openmetadata.csv.CsvUtil.addEntityReferences; +import static org.openmetadata.csv.CsvUtil.addField; +import static org.openmetadata.csv.CsvUtil.addTagLabels; import com.fasterxml.jackson.core.JsonProcessingException; import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.CSVRecord; +import org.openmetadata.csv.CsvUtil; +import org.openmetadata.csv.EntityCsv; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.api.data.TermReference; import org.openmetadata.schema.entity.data.Glossary; +import org.openmetadata.schema.entity.data.GlossaryTerm; import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.ProviderType; import org.openmetadata.schema.type.Relationship; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TagLabel.TagSource; +import org.openmetadata.schema.type.csv.CsvHeader; +import org.openmetadata.schema.type.csv.CsvImportResult; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; @@ -96,6 +115,123 @@ public class GlossaryRepository extends EntityRepository { return new GlossaryUpdater(original, updated, operation); } + @Override + public String exportToCsv(String name, String user) throws IOException { + Glossary glossary = getByName(null, name, Fields.EMPTY_FIELDS); // Validate glossary name + EntityRepository repository = Entity.getEntityRepository(Entity.GLOSSARY_TERM); + ListFilter filter = new ListFilter(Include.NON_DELETED).addQueryParam("parent", name); + List terms = repository.listAll(repository.getFields("reviewers,tags,relatedTerms"), filter); + terms.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName)); + return new GlossaryCsv(glossary, user).exportCsv(terms); + } + + /** Load CSV provided for bulk upload */ + @Override + public CsvImportResult importFromCsv(String name, String csv, boolean dryRun, String user) throws IOException { + Glossary glossary = getByName(null, name, Fields.EMPTY_FIELDS); // Validate glossary name + GlossaryCsv glossaryCsv = new GlossaryCsv(glossary, user); + return glossaryCsv.importCsv(csv, dryRun); + } + + public static class GlossaryCsv extends EntityCsv { + public static final List HEADERS = new ArrayList<>(); + private final Glossary glossary; + + static { + HEADERS.add(new CsvHeader().withName("parent").withRequired(false)); + HEADERS.add(new CsvHeader().withName("name").withRequired(true)); + HEADERS.add(new CsvHeader().withName("displayName").withRequired(false)); + HEADERS.add(new CsvHeader().withName("description").withRequired(true)); + HEADERS.add(new CsvHeader().withName("synonyms").withRequired(false)); + HEADERS.add(new CsvHeader().withName("relatedTerms").withRequired(false)); + HEADERS.add(new CsvHeader().withName("references").withRequired(false)); + HEADERS.add(new CsvHeader().withName("tags").withRequired(false)); + } + + GlossaryCsv(Glossary glossary, String user) { + super(Entity.GLOSSARY_TERM, HEADERS, user); + this.glossary = glossary; + } + + @Override + protected GlossaryTerm toEntity(CSVPrinter printer, CSVRecord record) throws IOException { + GlossaryTerm glossaryTerm = new GlossaryTerm().withGlossary(glossary.getEntityReference()); + + // Field 1 - parent term + glossaryTerm.withParent(getEntityReference(printer, record, 0, Entity.GLOSSARY_TERM)); + if (!processRecord) { + return null; + } + + // Field 2,3,4 - Glossary name, displayName, description + glossaryTerm.withName(record.get(1)).withDisplayName(record.get(2)).withDescription(record.get(3)); + + // Field 5 - Synonym list + glossaryTerm.withSynonyms(CsvUtil.fieldToStrings(record.get(4))); + + // Field 6 - Related terms + glossaryTerm.withRelatedTerms(getEntityReferences(printer, record, 5, Entity.GLOSSARY_TERM)); + if (!processRecord) { + return null; + } + + // Field 7 - TermReferences + glossaryTerm.withReferences(getTermReferences(printer, record, 6)); + if (!processRecord) { + return null; + } + + // Field 8 - tags + glossaryTerm.withTags(getTagLabels(printer, record, 7)); + if (!processRecord) { + return null; + } + return glossaryTerm; + } + + private List getTermReferences(CSVPrinter printer, CSVRecord record, int fieldNumber) + throws IOException { + String termRefs = record.get(fieldNumber); + if (nullOrEmpty(termRefs)) { + return null; + } + List termRefList = CsvUtil.fieldToStrings(termRefs); + if (termRefList.size() % 2 != 0) { + // List should have even numbered terms - termName and endPoint + importFailure(printer, invalidField(fieldNumber, "Term references should termName;endpoint"), record); + processRecord = false; + return null; + } + List list = new ArrayList<>(); + for (int i = 0; i < termRefList.size(); ) { + list.add(new TermReference().withName(termRefList.get(i++)).withEndpoint(URI.create(termRefList.get(i++)))); + } + return list; + } + + @Override + protected List toRecord(GlossaryTerm entity) { + List record = new ArrayList<>(); + addEntityReference(record, entity.getParent()); + addField(record, entity.getName()); + addField(record, entity.getDisplayName()); + addField(record, entity.getDescription()); + CsvUtil.addFieldList(record, entity.getSynonyms()); + addEntityReferences(record, entity.getRelatedTerms()); + addField(record, termReferencesToRecord(entity.getReferences())); + addTagLabels(record, entity.getTags()); + return record; + } + + private String termReferencesToRecord(List list) { + return nullOrEmpty(list) + ? null + : list.stream() + .map(termReference -> termReference.getName() + CsvUtil.FIELD_SEPARATOR + termReference.getEndpoint()) + .collect(Collectors.joining(";")); + } + } + private List getReviewers(Glossary entity) throws IOException { List ids = findFrom(entity.getId(), Entity.GLOSSARY, Relationship.REVIEWS, Entity.USER); return EntityUtil.populateEntityReferences(ids, Entity.USER); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java index 47f4a0e7bb3..cc519f321bf 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java @@ -19,6 +19,7 @@ import org.openmetadata.schema.type.EntityHistory; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.type.csv.CsvImportResult; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.jdbi3.EntityRepository; @@ -252,6 +253,19 @@ public abstract class EntityResource expectedRecord = new ArrayList<>(); + List actualRecord = new ArrayList<>(); + + // Add string + expectedRecord.add(""); + assertEquals(expectedRecord, CsvUtil.addField(actualRecord, null)); + + expectedRecord.add("abc"); + assertEquals(expectedRecord, CsvUtil.addField(actualRecord, "abc")); + + // Add list of strings + expectedRecord.add(""); + assertEquals(expectedRecord, CsvUtil.addFieldList(actualRecord, null)); + + expectedRecord.add("def;ghi"); + assertEquals(expectedRecord, CsvUtil.addFieldList(actualRecord, listOf("def", "ghi"))); + + // Add entity reference + expectedRecord.add(""); + assertEquals(expectedRecord, CsvUtil.addEntityReference(actualRecord, null)); // Null entity reference + + expectedRecord.add("fqn"); + assertEquals( + expectedRecord, CsvUtil.addEntityReference(actualRecord, new EntityReference().withFullyQualifiedName("fqn"))); + + // Add entity references + expectedRecord.add(""); + assertEquals(expectedRecord, CsvUtil.addEntityReferences(actualRecord, null)); // Null entity references + + expectedRecord.add("fqn1;fqn2"); + List refs = + listOf( + new EntityReference().withFullyQualifiedName("fqn1"), new EntityReference().withFullyQualifiedName("fqn2")); + assertEquals(expectedRecord, CsvUtil.addEntityReferences(actualRecord, refs)); + + // Add tag labels + expectedRecord.add(""); + assertEquals(expectedRecord, CsvUtil.addTagLabels(actualRecord, null)); // Null entity references + + expectedRecord.add("t1;t2"); + List tags = listOf(new TagLabel().withTagFQN("t1"), new TagLabel().withTagFQN("t2")); + assertEquals(expectedRecord, CsvUtil.addTagLabels(actualRecord, tags)); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java b/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java new file mode 100644 index 00000000000..385cc230f71 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java @@ -0,0 +1,173 @@ +package org.openmetadata.csv; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.openmetadata.common.utils.CommonUtil.listOf; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.csv.CsvUtil.LINE_SEPARATOR; +import static org.openmetadata.csv.CsvUtil.recordToString; +import static org.openmetadata.csv.EntityCsv.ENTITY_CREATED; +import static org.openmetadata.csv.EntityCsv.ENTITY_UPDATED; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.CSVRecord; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.type.csv.CsvFile; +import org.openmetadata.schema.type.csv.CsvHeader; +import org.openmetadata.schema.type.csv.CsvImportResult; +import org.openmetadata.schema.type.csv.CsvImportResult.Status; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO.TableDAO; +import org.openmetadata.service.jdbi3.TableRepository; + +public class EntityCsvTest { + private static final List CSV_HEADERS; + private static final String HEADER_STRING = "h1*,h2,h3" + LINE_SEPARATOR; + + static { + Object[][] headers = { + {"h1", Boolean.TRUE}, + {"h2", Boolean.FALSE}, + {"h3", Boolean.FALSE} + }; + CSV_HEADERS = getHeaders(headers); + } + + @BeforeAll + public static void setup() { + Entity.registerEntity(Table.class, Entity.TABLE, Mockito.mock(TableDAO.class), Mockito.mock(TableRepository.class)); + } + + @Test + void test_formatHeader() throws IOException { + CsvFile csvFile = new CsvFile(); + csvFile.withHeaders(CSV_HEADERS); + String file = CsvUtil.formatCsv(csvFile); + assertEquals(HEADER_STRING, file); + } + + @Test + void test_validateCsvInvalidHeader() throws IOException { + String csv = ",h2,h3" + LINE_SEPARATOR; // Header h1 is missing in the CSV file + TestCsv testCsv = new TestCsv(CSV_HEADERS); + CsvImportResult importResult = testCsv.importCsv(csv, true); + assertSummary(importResult, Status.ABORTED, 1, 0, 1); + assertNull(importResult.getImportResultsCsv()); + assertEquals(TestCsv.invalidHeader("h1*,h2,h3", ",h2,h3"), importResult.getAbortReason()); + } + + @Test + void test_validateCsvInvalidRecords() throws IOException { + // Invalid record 2 - Missing required value in h1 + // Invalid record 3 - Record with only two fields instead of 3 + List records = listOf(",2,3", "1,2", "1,2,3"); + String csv = createCsv(CSV_HEADERS, records); + + TestCsv testCsv = new TestCsv(CSV_HEADERS); + CsvImportResult importResult = testCsv.importCsv(csv, true); + assertSummary(importResult, Status.PARTIAL_SUCCESS, 4, 2, 2); + + String[] expectedRecords = { + CsvUtil.recordToString(EntityCsv.getResultHeaders(CSV_HEADERS)), + getFailedRecord(",2,3", TestCsv.fieldRequired(0)), + getFailedRecord("1,2", TestCsv.invalidFieldCount(3, 2)), + getSuccessRecord("1,2,3", ENTITY_CREATED) + }; + + assertRows(importResult, expectedRecords); + } + + public static void assertSummary( + CsvImportResult importResult, + Status expectedStatus, + int expectedRowsProcessed, + int expectedRowsPassed, + int expectedRowsFailed) { + assertEquals(expectedStatus, importResult.getStatus()); + assertEquals(expectedRowsProcessed, importResult.getNumberOfRowsProcessed()); + assertEquals(expectedRowsPassed, importResult.getNumberOfRowsPassed()); + assertEquals(expectedRowsFailed, importResult.getNumberOfRowsFailed()); + } + + public static void assertRows(CsvImportResult importResult, String... expectedRows) { + String[] resultRecords = importResult.getImportResultsCsv().split(LINE_SEPARATOR); + assertEquals(expectedRows.length, resultRecords.length); + for (int i = 0; i < resultRecords.length; i++) { + assertEquals(expectedRows[i], resultRecords[i], "Row number is " + i); + } + } + + public static String getSuccessRecord(String record, String successDetails) { + return String.format("%s,%s,%s", EntityCsv.IMPORT_STATUS_SUCCESS, successDetails, record); + } + + public static String getFailedRecord(String record, String errorDetails) { + return String.format("%s,\"%s\",%s", EntityCsv.IMPORT_STATUS_FAILED, errorDetails, record); + } + + private static List getHeaders(Object[][] headers) { + List csvHeaders = new ArrayList<>(); + for (Object[] header : headers) { + csvHeaders.add(new CsvHeader().withName((String) header[0]).withRequired((Boolean) header[1])); + } + return csvHeaders; + } + + public static String createCsv(List csvHeaders, List records) { + records.add(0, recordToString(CsvUtil.getHeaders(csvHeaders))); + return String.join(LINE_SEPARATOR, records) + LINE_SEPARATOR; + } + + public static String createCsv(List csvHeaders, List createRecords, List updateRecords) { + // Create CSV + List csvRecords = new ArrayList<>(); + if (!nullOrEmpty(createRecords)) { + csvRecords.addAll(createRecords); + } + if (!nullOrEmpty(updateRecords)) { + csvRecords.addAll(updateRecords); + } + return createCsv(csvHeaders, csvRecords); + } + + public static String createCsvResult( + List csvHeaders, List createRecords, List updateRecords) { + // Create CSV + List csvRecords = new ArrayList<>(); + csvRecords.add(recordToString(EntityCsv.getResultHeaders(csvHeaders))); + if (!nullOrEmpty(createRecords)) { + for (String record : createRecords) { + csvRecords.add(getSuccessRecord(record, ENTITY_CREATED)); + } + } + if (!nullOrEmpty(updateRecords)) { + for (String record : updateRecords) { + csvRecords.add(getSuccessRecord(record, ENTITY_UPDATED)); + } + } + return String.join(LINE_SEPARATOR, csvRecords) + LINE_SEPARATOR; + } + + private static class TestCsv extends EntityCsv { + protected TestCsv(List csvHeaders) { + super(Entity.TABLE, csvHeaders, "admin"); + } + + @Override + protected EntityInterface toEntity(CSVPrinter resultsPrinter, CSVRecord record) { + return new Table(); // Return a random entity to mark successfully processing a record + } + + @Override + protected List toRecord(EntityInterface entity) { + return null; + } + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index 06764d321f4..ce21a19cb9a 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -16,9 +16,17 @@ package org.openmetadata.service.resources.glossary; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.csv.CsvUtil.recordToString; +import static org.openmetadata.csv.EntityCsv.entityNotFound; +import static org.openmetadata.csv.EntityCsvTest.assertRows; +import static org.openmetadata.csv.EntityCsvTest.assertSummary; +import static org.openmetadata.csv.EntityCsvTest.createCsv; +import static org.openmetadata.csv.EntityCsvTest.getFailedRecord; import static org.openmetadata.schema.type.ProviderType.SYSTEM; import static org.openmetadata.service.util.EntityUtil.fieldAdded; import static org.openmetadata.service.util.EntityUtil.fieldDeleted; @@ -36,6 +44,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Response.Status; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpResponseException; @@ -43,7 +52,8 @@ import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.TestMethodOrder; -import org.openmetadata.common.utils.CommonUtil; +import org.openmetadata.csv.EntityCsv; +import org.openmetadata.csv.EntityCsvTest; import org.openmetadata.schema.api.data.CreateGlossary; import org.openmetadata.schema.api.data.CreateGlossaryTerm; import org.openmetadata.schema.api.data.CreateTable; @@ -56,8 +66,11 @@ import org.openmetadata.schema.type.ColumnDataType; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.ProviderType; import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.schema.type.csv.CsvHeader; +import org.openmetadata.schema.type.csv.CsvImportResult; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.jdbi3.GlossaryRepository.GlossaryCsv; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.databases.TableResourceTest; import org.openmetadata.service.util.EntityUtil; @@ -177,7 +190,7 @@ public class GlossaryResourceTest extends EntityResourceTest tagLabels = toTagLabels(t1, t11, t12, t2, t21, t22); Column column = new Column().withName("c1").withDataType(ColumnDataType.INT).withTags(tagLabels); CreateTable createTable = - tableResourceTest.createRequest(getEntityName(test)).withTags(tagLabels).withColumns(CommonUtil.listOf(column)); + tableResourceTest.createRequest(getEntityName(test)).withTags(tagLabels).withColumns(listOf(column)); Table table = tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS); // @@ -234,7 +247,7 @@ public class GlossaryResourceTest extends EntityResourceTest tagLabels = toTagLabels(t1, t11, t111, t12, t121, t13, t131, t2, t21, t211, h1, h11, h111); Column column = new Column().withName("c1").withDataType(ColumnDataType.INT).withTags(tagLabels); CreateTable createTable = - tableResourceTest.createRequest(getEntityName(test)).withTags(tagLabels).withColumns(CommonUtil.listOf(column)); + tableResourceTest.createRequest(getEntityName(test)).withTags(tagLabels).withColumns(listOf(column)); Table table = tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS); Object[][] scenarios = { @@ -287,6 +300,92 @@ public class GlossaryResourceTest extends EntityResourceTest createRecords = + listOf( + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1", + ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,Tier.Tier2"); + importCsvAndValidate(glossaryName, GlossaryCsv.HEADERS, createRecords, null); // Dry run + + // Update terms with change in description + List updateRecords = + listOf( + ",g1,dsp1,new-dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1", + ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,Tier.Tier2"); + importCsvAndValidate(glossaryName, GlossaryCsv.HEADERS, null, updateRecords); + + // Add new row to existing rows + createRecords = listOf(",g0,dsp0,dsc0,h1;h2;h3,,term0;http://term0,Tier.Tier3"); + importCsvAndValidate(glossaryName, GlossaryCsv.HEADERS, createRecords, updateRecords); + } + + private void importCsvAndValidate( + String glossaryName, List csvHeaders, List createRecords, List updateRecords) + throws HttpResponseException { + createRecords = listOrEmpty(createRecords); + updateRecords = listOrEmpty(updateRecords); + + // Import CSV with dryRun=true first + String csv = EntityCsvTest.createCsv(csvHeaders, createRecords, updateRecords); + CsvImportResult dryRunResult = importCsv(glossaryName, csv, true); + + // Validate the import result summary + int totalRows = 1 + createRecords.size() + updateRecords.size(); + assertSummary(dryRunResult, CsvImportResult.Status.SUCCESS, totalRows, totalRows, 0); + + // Validate the import result CSV + String resultsCsv = EntityCsvTest.createCsvResult(csvHeaders, createRecords, updateRecords); + assertEquals(resultsCsv, dryRunResult.getImportResultsCsv()); + + // Import CSV with dryRun=false to import the data + CsvImportResult result = importCsv(glossaryName, csv, false); + assertEquals(dryRunResult.withDryRun(false), result); + + // Finally, export CSV and ensure the exported CSV is same as imported CSV + String exportedCsv = exportCsv(glossaryName); + assertEquals(csv, exportedCsv); + } + + private CsvImportResult importCsv(String glossaryName, String csv, boolean dryRun) throws HttpResponseException { + WebTarget target = getResourceByName(glossaryName + "/import"); + target = !dryRun ? target.queryParam("dryRun", false) : target; + return TestUtils.putCsv(target, csv, CsvImportResult.class, Status.OK, ADMIN_AUTH_HEADERS); + } + + private String exportCsv(String glossaryName) throws HttpResponseException { + WebTarget target = getResourceByName(glossaryName + "/export"); + return TestUtils.get(target, String.class, ADMIN_AUTH_HEADERS); + } + private void copyGlossaryTerm(GlossaryTerm from, GlossaryTerm to) { to.withGlossary(from.getGlossary()) .withParent(from.getParent()) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java index 0c87859542c..961a2c5cc60 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java @@ -341,6 +341,14 @@ public final class TestUtils { return readResponse(response, clz, expectedStatus.getStatusCode()); } + public static T putCsv( + WebTarget target, String request, Class clz, Status expectedStatus, Map headers) + throws HttpResponseException { + Response response = + SecurityUtil.addHeaders(target, headers).method("PUT", Entity.entity(request, MediaType.TEXT_PLAIN)); + return readResponse(response, clz, expectedStatus.getStatusCode()); + } + public static void get(WebTarget target, Map headers) throws HttpResponseException { final Response response = SecurityUtil.addHeaders(target, headers).get(); readResponse(response, Status.NO_CONTENT.getStatusCode()); diff --git a/openmetadata-spec/src/main/resources/json/schema/type/basic.json b/openmetadata-spec/src/main/resources/json/schema/type/basic.json index f4667c0bdc8..aed38926a3f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/basic.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/basic.json @@ -140,6 +140,11 @@ "additionalProperties": { "type": "string" } + }, + "status" : { + "description": "State of an action over API.", + "type" : "string", + "enum" : ["success", "failure", "aborted", "partialSuccess"] } } } diff --git a/openmetadata-spec/src/main/resources/json/schema/type/csvErrorType.json b/openmetadata-spec/src/main/resources/json/schema/type/csvErrorType.json new file mode 100644 index 00000000000..25deec76430 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/type/csvErrorType.json @@ -0,0 +1,16 @@ +{ + "$id": "https://open-metadata.org/schema/type/csvErrorType.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "csvErrorType", + "type": "string", + "javaType": "org.openmetadata.schema.type.csv.CsvErrorType", + "description": "What type of error occurred when importing a CSV file.", + "enum": [ + "UNKNOWN", + "PARSER_FAILURE", + "INVALID_HEADER", + "INVALID_FIELD_COUNT", + "FIELD_REQUIRED", + "INVALID_FIELD" + ] +} diff --git a/openmetadata-spec/src/main/resources/json/schema/type/csvFile.json b/openmetadata-spec/src/main/resources/json/schema/type/csvFile.json new file mode 100644 index 00000000000..b0cd99d5694 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/type/csvFile.json @@ -0,0 +1,48 @@ +{ + "$id": "https://open-metadata.org/schema/type/csvFile.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "csvFile", + "description": "Represents a CSV file.", + "javaType": "org.openmetadata.schema.type.csv.CsvFile", + "definitions": { + "csvHeader": { + "description": "Represents a header for a field in a CSV file.", + "javaType": "org.openmetadata.schema.type.csv.CsvHeader", + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "required": { + "type": "boolean", + "default": false + } + }, + "additionalProperties": false + }, + "csvRecord": { + "javaType": "org.openmetadata.schema.type.csv.CsvRecord", + "description": "Represents a CSV record that contains one row values separated by a separator.", + "type": "array", + "items" : { + "type" : "string" + } + } + }, + "type": "object", + "properties": { + "headers": { + "type": "array", + "items": { + "$ref": "#/definitions/csvHeader" + } + }, + "records": { + "type": "array", + "items": { + "$ref": "#/definitions/csvRecord" + } + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/type/csvImportResult.json b/openmetadata-spec/src/main/resources/json/schema/type/csvImportResult.json new file mode 100644 index 00000000000..31ad35dd1cb --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/type/csvImportResult.json @@ -0,0 +1,50 @@ +{ + "$id": "https://open-metadata.org/schema/type/csvImportResult.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "csvImportResult", + "description": "Represents result of importing a CSV file. Detailed error is provided on if the CSV file is conformant and failure to load any of the records in the CSV file.", + "type": "object", + "javaType": "org.openmetadata.schema.type.csv.CsvImportResult", + "definitions": { + "rowCount" : { + "description": "Type used to indicate row count", + "type": "integer", + "format" : "int64", + "minimum": 0, + "default": 0 + }, + "index" : { + "description": "Type used to indicate row number or field number. In CSV the indexes start with 1.", + "type": "integer", + "format" : "int64", + "minimum": 1 + } + }, + "properties": { + "dryRun" : { + "description": "True if the CSV import has dryRun flag enabled", + "type" : "boolean" + }, + "status" : { + "$ref" : "basic.json#/definitions/status" + }, + "abortReason": { + "description": "Reason why import was aborted. This is set only when the `status` field is set to `aborted`", + "type" : "string" + }, + "numberOfRowsProcessed" : { + "$ref": "#/definitions/rowCount" + }, + "numberOfRowsPassed" : { + "$ref": "#/definitions/rowCount" + }, + "numberOfRowsFailed" : { + "$ref": "#/definitions/rowCount" + }, + "importResultsCsv" : { + "description": "CSV file that captures the result of import operation.", + "type" : "string" + } + }, + "additionalProperties": false +} \ No newline at end of file