diff --git a/backend-service/app/actors/TreeBuilderActor.java b/backend-service/app/actors/TreeBuilderActor.java index 49079f11ec..9df62ba6fb 100644 --- a/backend-service/app/actors/TreeBuilderActor.java +++ b/backend-service/app/actors/TreeBuilderActor.java @@ -53,7 +53,7 @@ public class TreeBuilderActor extends UntypedActor { in = EtlJob.class.getClassLoader().getResourceAsStream("jython/DatasetTreeBuilder.py"); break; case "flow": - in = EtlJob.class.getClassLoader().getResourceAsStream("jython/FlowTreeBuilder.py"); + //in = EtlJob.class.getClassLoader().getResourceAsStream("jython/FlowTreeBuilder.py"); break; default: Logger.error("unknown message : {}", msg); diff --git a/metadata-etl/src/main/java/metadata/etl/models/EtlJobFactory.java b/metadata-etl/src/main/java/metadata/etl/models/EtlJobFactory.java index e443600541..5a8edc5020 100644 --- a/metadata-etl/src/main/java/metadata/etl/models/EtlJobFactory.java +++ b/metadata-etl/src/main/java/metadata/etl/models/EtlJobFactory.java @@ -55,6 +55,8 @@ public class EtlJobFactory { return new HiveMetadataEtl(refId, whExecId, properties); case ELASTICSEARCH_EXECUTION_INDEX_ETL: return new ElasticSearchBuildIndexETL(refId, whExecId, properties); + case TREEBUILDER_EXECUTION_DATASET_ETL: + return new ElasticSearchBuildIndexETL(refId, whExecId, properties); default: throw new UnsupportedOperationException("Unsupported job type: " + etlJobName); } diff --git a/metadata-etl/src/main/java/metadata/etl/models/EtlJobName.java b/metadata-etl/src/main/java/metadata/etl/models/EtlJobName.java index 67432702e2..e0b075ed3b 100644 --- a/metadata-etl/src/main/java/metadata/etl/models/EtlJobName.java +++ b/metadata-etl/src/main/java/metadata/etl/models/EtlJobName.java @@ -27,6 +27,7 @@ public enum EtlJobName { GIT_MEDATA_ETL(EtlType.VCS, RefIdType.APP), HIVE_DATASET_METADATA_ETL(EtlType.DATASET, RefIdType.DB), ELASTICSEARCH_EXECUTION_INDEX_ETL(EtlType.OPERATION, RefIdType.APP), + TREEBUILDER_EXECUTION_DATASET_ETL(EtlType.OPERATION, RefIdType.APP), ; EtlType etlType; diff --git a/metadata-etl/src/main/java/metadata/etl/treebuilder/DatasetTreeBuildETL.java b/metadata-etl/src/main/java/metadata/etl/treebuilder/DatasetTreeBuildETL.java new file mode 100644 index 0000000000..ffcdb2316b --- /dev/null +++ b/metadata-etl/src/main/java/metadata/etl/treebuilder/DatasetTreeBuildETL.java @@ -0,0 +1,52 @@ +/** + * Copyright 2015 LinkedIn Corp. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ +package metadata.etl.treebuilder; + +import java.io.InputStream; +import java.util.Properties; +import metadata.etl.EtlJob; + + +public class DatasetTreeBuildETL extends EtlJob { + + public DatasetTreeBuildETL(int appId, long whExecId) { + super(appId, null, whExecId); + } + + public DatasetTreeBuildETL(int appId, long whExecId, Properties properties) { + super(appId, null, whExecId, properties); + } + + @Override + public void extract() + throws Exception { + logger.info("In DatasetTreeBuildETL java launch extract jython scripts"); + } + + @Override + public void transform() + throws Exception { + logger.info("In DatasetTreeBuildETL java launch transform jython scripts"); + } + + @Override + public void load() + throws Exception { + logger.info("In DatasetTreeBuildETL java launch load jython scripts"); + InputStream inputStream = classLoader.getResourceAsStream("jython/DatasetTreeBuilder.py"); + interpreter.execfile(inputStream); + inputStream.close(); + logger.info("In DatasetTreeBuildETL java load jython scripts finished"); + } +} diff --git a/metadata-etl/src/test/java/metadata/etl/treebuilder/DatasetTreeBuildETLTest.java b/metadata-etl/src/test/java/metadata/etl/treebuilder/DatasetTreeBuildETLTest.java new file mode 100644 index 0000000000..665bc4bc06 --- /dev/null +++ b/metadata-etl/src/test/java/metadata/etl/treebuilder/DatasetTreeBuildETLTest.java @@ -0,0 +1,49 @@ +/** + * Copyright 2015 LinkedIn Corp. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + */ +package metadata.etl.treebuilder; + +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + + + +public class DatasetTreeBuildETLTest{ + DatasetTreeBuildETL datasetTreeBuildETL; + + @BeforeTest + public void setUp() + throws Exception { + datasetTreeBuildETL = new DatasetTreeBuildETL(40, 0L); + } + + @Test(groups = {"needConfig"}) + public void testExtract() throws Exception { + datasetTreeBuildETL.extract(); + } + + @Test(groups = {"needConfig"}) + public void testTransform() throws Exception { + datasetTreeBuildETL.transform(); + } + + @Test(groups = {"needConfig"}) + public void testLoad() throws Exception { + datasetTreeBuildETL.load(); + } + + @Test(groups = {"needConfig"}) + public void testRun() throws Exception { + datasetTreeBuildETL.run(); + } +}