yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-22 08:08:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #145 from jerrybai2009/master

Browse Source 
comment out the flow tree builder since UI does not use it anymore
This commit is contained in:
jerrybai2009 2016-06-14 16:17:03 -07:00 committed by GitHub
commit a1228aa46c
5 changed files with 105 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend-service/app/actors/TreeBuilderActor.java
View File

@ -53,7 +53,7 @@ public class TreeBuilderActor extends UntypedActor {
in = EtlJob.class.getClassLoader().getResourceAsStream("jython/DatasetTreeBuilder.py");
break;
case "flow":
in = EtlJob.class.getClassLoader().getResourceAsStream("jython/FlowTreeBuilder.py");
//in = EtlJob.class.getClassLoader().getResourceAsStream("jython/FlowTreeBuilder.py");
break;
default:
Logger.error("unknown message : {}", msg);

2
metadata-etl/src/main/java/metadata/etl/models/EtlJobFactory.java
View File

@ -55,6 +55,8 @@ public class EtlJobFactory {
return new HiveMetadataEtl(refId, whExecId, properties);
case ELASTICSEARCH_EXECUTION_INDEX_ETL:
return new ElasticSearchBuildIndexETL(refId, whExecId, properties);
case TREEBUILDER_EXECUTION_DATASET_ETL:
return new ElasticSearchBuildIndexETL(refId, whExecId, properties);
default:
throw new UnsupportedOperationException("Unsupported job type: " + etlJobName);
}

1
metadata-etl/src/main/java/metadata/etl/models/EtlJobName.java
View File

@ -27,6 +27,7 @@ public enum EtlJobName {
GIT_MEDATA_ETL(EtlType.VCS, RefIdType.APP),
HIVE_DATASET_METADATA_ETL(EtlType.DATASET, RefIdType.DB),
ELASTICSEARCH_EXECUTION_INDEX_ETL(EtlType.OPERATION, RefIdType.APP),
TREEBUILDER_EXECUTION_DATASET_ETL(EtlType.OPERATION, RefIdType.APP),
;
EtlType etlType;

52
metadata-etl/src/main/java/metadata/etl/treebuilder/DatasetTreeBuildETL.java Normal file
View File

@ -0,0 +1,52 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package metadata.etl.treebuilder;
import java.io.InputStream;
import java.util.Properties;
import metadata.etl.EtlJob;
public class DatasetTreeBuildETL extends EtlJob {
public DatasetTreeBuildETL(int appId, long whExecId) {
super(appId, null, whExecId);
}
public DatasetTreeBuildETL(int appId, long whExecId, Properties properties) {
super(appId, null, whExecId, properties);
}
@Override
public void extract()
throws Exception {
logger.info("In DatasetTreeBuildETL java launch extract jython scripts");
}
@Override
public void transform()
throws Exception {
logger.info("In DatasetTreeBuildETL java launch transform jython scripts");
}
@Override
public void load()
throws Exception {
logger.info("In DatasetTreeBuildETL java launch load jython scripts");
InputStream inputStream = classLoader.getResourceAsStream("jython/DatasetTreeBuilder.py");
interpreter.execfile(inputStream);
inputStream.close();
logger.info("In DatasetTreeBuildETL java load jython scripts finished");
}
}

49
metadata-etl/src/test/java/metadata/etl/treebuilder/DatasetTreeBuildETLTest.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Copyright 2015 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/
package metadata.etl.treebuilder;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
public class DatasetTreeBuildETLTest{
DatasetTreeBuildETL datasetTreeBuildETL;
@BeforeTest
public void setUp()
throws Exception {
datasetTreeBuildETL = new DatasetTreeBuildETL(40, 0L);
}
@Test(groups = {"needConfig"})
public void testExtract() throws Exception {
datasetTreeBuildETL.extract();
}
@Test(groups = {"needConfig"})
public void testTransform() throws Exception {
datasetTreeBuildETL.transform();
}
@Test(groups = {"needConfig"})
public void testLoad() throws Exception {
datasetTreeBuildETL.load();
}
@Test(groups = {"needConfig"})
public void testRun() throws Exception {
datasetTreeBuildETL.run();
}
}