yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-24 09:08:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Check Linkedin interanl flag before fetching schema from Metastore (#440)

Browse Source 
Also during column data mapping, check if native data type and description exists in the returned schema fields from Metadata store.
This commit is contained in:
Yi (Alan) Wang 2017-04-14 14:13:32 -07:00 committed by Mars Lan
parent fe0d2ea5f9
commit 6facc830cc
2 changed files with 22 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
web/app/controllers/api/v1/Dataset.java
View File

@ -25,6 +25,7 @@ import models.DatasetDependency;
import models.DatasetSecurity; import models.DatasetSecurity;
import models.ImpactDataset; import models.ImpactDataset;
import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.lang3.math.NumberUtils;
import play.Play;
import play.libs.F.Promise; import play.libs.F.Promise;
import play.libs.Json; import play.libs.Json;
import play.mvc.Controller; import play.mvc.Controller;
@ -40,6 +41,9 @@ import java.util.Map;
public class Dataset extends Controller public class Dataset extends Controller
{ {
private static final Boolean
LINKEDIN_INTERNAL = Play.application().configuration().getBoolean("linkedin.internal", false);
public static Result getDatasetOwnerTypes() public static Result getDatasetOwnerTypes()
{ {
ObjectNode result = Json.newObject(); ObjectNode result = Json.newObject();
@ -151,23 +155,25 @@ public class Dataset extends Controller
} }
// try to access dataset fields from Metadata Store schemaMetadata first // try to access dataset fields from Metadata Store schemaMetadata first
SchemaFieldArray datasetFields = null; if (LINKEDIN_INTERNAL) {
try { try {
datasetFields = MetadataStoreDao.getLatestSchemaByWhUrn(urn).getFields(); SchemaFieldArray datasetFields = MetadataStoreDao.getLatestSchemaByWhUrn(urn).getFields();
} catch (Exception e) {
Logger.debug("Can't find schema for URN: " + urn + ", Exception: " + e.getMessage()); if (datasetFields != null && datasetFields.size() > 0) {
List<DatasetColumn> columns = MetadataStoreDao.datasetColumnsMapper(datasetFields);
result.put("status", "ok");
result.set("columns", Json.toJson(columns));
return ok(result);
}
} catch (Exception e) {
Logger.debug("Fail to fetch schema from Metadata-store for URN: " + urn + ", Exception: " + e.getMessage());
}
} }
List<DatasetColumn> datasetColumnList; List<DatasetColumn> columns = DatasetsDAO.getDatasetColumnsByID(id);
if (datasetFields != null && datasetFields.size() > 0) { if (columns != null && columns.size() > 0) {
datasetColumnList = MetadataStoreDao.datasetColumnsMapper(datasetFields);
} else { // get fields from WH db if nothing in Metadata store
datasetColumnList = DatasetsDAO.getDatasetColumnsByID(id);
}
if (datasetColumnList != null && datasetColumnList.size() > 0) {
result.put("status", "ok"); result.put("status", "ok");
result.set("columns", Json.toJson(datasetColumnList)); result.set("columns", Json.toJson(columns));
} else { } else {
result.put("status", "error"); result.put("status", "error");
result.put("message", "record not found"); result.put("message", "record not found");

4
web/app/dao/MetadataStoreDao.java
View File

@ -49,8 +49,8 @@ public class MetadataStoreDao {
DatasetColumn col = new DatasetColumn(); DatasetColumn col = new DatasetColumn();
col.fieldName = field.getFieldPath(); col.fieldName = field.getFieldPath();
col.fullFieldPath = field.getFieldPath(); col.fullFieldPath = field.getFieldPath();
col.dataType = field.getNativeDataType(); col.dataType = field.hasNativeDataType() ? field.getNativeDataType() : "";
col.comment = field.getDescription(); col.comment = field.hasDescription() ? field.getDescription() : "";
columns.add(col); columns.add(col);
} }
return columns; return columns;