yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-09 16:02:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix dataset insert API bug. Fix load sql bug.

Browse Source
This commit is contained in:
SunZhaonan 2016-03-18 15:26:55 -07:00
parent 327937486b
commit c66b00e2f6
4 changed files with 13 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend-service/app/actors/TreeBuilderActor.java
View File

@ -56,14 +56,14 @@ public class TreeBuilderActor extends UntypedActor {
in = EtlJob.class.getClassLoader().getResourceAsStream("jython/FlowTreeBuilder.py");
break;
default:
Logger.warn("unknown message : {}", msg);
Logger.error("unknown message : {}", msg);
}
if (in != null) {
interpreter.execfile(in);
in.close();
Logger.info("Finish build {} tree", msg);
} else {
Logger.warn("can not find jython script");
Logger.error("can not find jython script");
}
} else {
throw new Exception("message type is not supported!");

12
backend-service/app/models/daos/DatasetDao.java
View File

@ -54,13 +54,15 @@ public class DatasetDao {
ObjectMapper om = new ObjectMapper();
om.setPropertyNamingStrategy(PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
DatasetRecord record = om.convertValue(dataset, DatasetRecord.class);
Map<String, Object> refDataset = getDatasetByUrn(record.getRefDatasetUrn());
// Find ref dataset id
if (refDataset != null) {
record.setRefDatasetId((int) refDataset.get("id"));
if (record.getRefDatasetUrn() != null) {
Map<String, Object> refDataset = getDatasetByUrn(record.getRefDatasetUrn());
// Find ref dataset id
if (refDataset != null) {
record.setRefDatasetId(((Long) refDataset.get("id")).intValue());
}
}
// Find layout id
if (record.getSamplePartitionFullPath() != null) {
PartitionPatternMatcher ppm = new PartitionPatternMatcher(PartitionLayoutDao.getPartitionLayouts());

4
metadata-etl/src/main/resources/jython/HdfsLoad.py
View File

@ -151,9 +151,9 @@ class HdfsLoad:
or description in ('null', 'N/A', 'nothing', 'empty', 'none'));
insert into field_comments (
user_id, comment, created, comment_crc32_checksum
user_id, comment, created, modified, comment_crc32_checksum
)
select 0 user_id, description, now() created, crc32(description) from
select 0 user_id, description, now() created, now() modified, crc32(description) from
(
select sf.description
from stg_dict_field_detail sf left join field_comments fc

4
metadata-etl/src/main/resources/jython/HiveLoad.py
View File

@ -185,9 +185,9 @@ class HiveLoad:
insert into field_comments (
user_id, comment, created, comment_crc32_checksum
user_id, comment, created, modified, comment_crc32_checksum
)
select 0 user_id, description, now() created, crc32(description) from
select 0 user_id, description, now() created, now() modified, crc32(description) from
(
select sf.description
from stg_dict_field_detail sf left join field_comments fc