diff --git a/wherehows-etl/src/main/java/metadata/etl/EtlJob.java b/wherehows-etl/src/main/java/metadata/etl/EtlJob.java index 18d3373177..a81013f1bb 100644 --- a/wherehows-etl/src/main/java/metadata/etl/EtlJob.java +++ b/wherehows-etl/src/main/java/metadata/etl/EtlJob.java @@ -45,30 +45,6 @@ public abstract class EtlJob { // default location of local test configuration file private final static String DEFAULT_CONFIG_FILE_LOCATION = System.getProperty("user.home") + "/.wherehows/local_test.properties"; - /** - * Constructor for using config file - * @param appId - * @param whExecId generated by backend service - */ - @Deprecated - public EtlJob(Integer appId, Integer dbId, long whExecId) { - this(appId, dbId, whExecId, DEFAULT_CONFIG_FILE_LOCATION); - } - - /** - * Private constructor for using config file - * @param appId - * @param dbId - * @param whExecId - * @param configFile - */ - @Deprecated - public EtlJob(Integer appId, Integer dbId, long whExecId, String configFile) { - PySystemState sys = configFromFile(appId, dbId, whExecId, configFile); - addJythonToPath(sys); - interpreter = new PythonInterpreter(null, sys); - } - /** * Used by backend service * @param appId nullable @@ -94,39 +70,6 @@ public abstract class EtlJob { } } - @Deprecated - private PySystemState configFromFile(Integer appId, Integer dbId, long whExecId, String configFile) { - - prop = new Properties(); - if (appId != null) { - prop.setProperty(Constant.APP_ID_KEY, String.valueOf(appId)); - } - if (dbId != null) { - prop.setProperty(Constant.DB_ID_KEY, String.valueOf(dbId)); - } - prop.setProperty(Constant.WH_EXEC_ID_KEY, String.valueOf(whExecId)); - - try { - InputStream propFile = new FileInputStream(configFile); - prop.load(propFile); - propFile.close(); - } catch (IOException e) { - logger.error("property file '{}' not found", configFile); - e.printStackTrace(); - } - - PyDictionary config = new PyDictionary(); - - for (String key : prop.stringPropertyNames()) { - String value = prop.getProperty(key); - config.put(new PyString(key), new PyString(value)); - } - - PySystemState sys = new PySystemState(); - sys.argv.append(config); - return sys; - } - /** * Copy all properties into jython envirenment * @param appId diff --git a/wherehows-etl/src/main/java/metadata/etl/dataset/hdfs/HdfsMetadataEtl.java b/wherehows-etl/src/main/java/metadata/etl/dataset/hdfs/HdfsMetadataEtl.java index 484e454883..8bb9479ab9 100644 --- a/wherehows-etl/src/main/java/metadata/etl/dataset/hdfs/HdfsMetadataEtl.java +++ b/wherehows-etl/src/main/java/metadata/etl/dataset/hdfs/HdfsMetadataEtl.java @@ -45,16 +45,6 @@ import wherehows.common.Constant; */ public class HdfsMetadataEtl extends EtlJob { - /** - * Constructor used in test - * @param dbId - * @param whExecId - */ - @Deprecated - public HdfsMetadataEtl(Integer dbId, Long whExecId) { - super(null, dbId, whExecId); - } - /** * Copy the jar to remote gateway, run the collecting job on remote, copy back the result. * @param dbId the database need to collect diff --git a/wherehows-etl/src/main/java/metadata/etl/dataset/hive/HiveMetadataEtl.java b/wherehows-etl/src/main/java/metadata/etl/dataset/hive/HiveMetadataEtl.java index 835f488c4f..ffc8aea56c 100644 --- a/wherehows-etl/src/main/java/metadata/etl/dataset/hive/HiveMetadataEtl.java +++ b/wherehows-etl/src/main/java/metadata/etl/dataset/hive/HiveMetadataEtl.java @@ -24,16 +24,10 @@ import wherehows.common.Constant; */ public class HiveMetadataEtl extends EtlJob { - @Deprecated - public HiveMetadataEtl(int dbId, long whExecId) { - super(null, dbId, whExecId); - } - public HiveMetadataEtl(int dbId, long whExecId, Properties prop) { super(null, dbId, whExecId, prop); } - @Override public void extract() throws Exception { diff --git a/wherehows-etl/src/main/java/metadata/etl/git/GitMetadataEtl.java b/wherehows-etl/src/main/java/metadata/etl/git/GitMetadataEtl.java index 0ebe0aefb0..997661bd18 100644 --- a/wherehows-etl/src/main/java/metadata/etl/git/GitMetadataEtl.java +++ b/wherehows-etl/src/main/java/metadata/etl/git/GitMetadataEtl.java @@ -36,10 +36,6 @@ public class GitMetadataEtl extends EtlJob { protected final Logger logger = LoggerFactory.getLogger(getClass()); public static final String COMMIT_OUTPUT_FILE = "commit.csv"; - public GitMetadataEtl(int appId, long whExecId) { - super(appId, null, whExecId); - } - public GitMetadataEtl(int appId, long whExecId, Properties prop) { super(appId, null, whExecId, prop); } diff --git a/wherehows-etl/src/main/java/metadata/etl/lineage/AzLineageMetadataEtl.java b/wherehows-etl/src/main/java/metadata/etl/lineage/AzLineageMetadataEtl.java index 7ed9ccfcc4..75ead8b9a6 100644 --- a/wherehows-etl/src/main/java/metadata/etl/lineage/AzLineageMetadataEtl.java +++ b/wherehows-etl/src/main/java/metadata/etl/lineage/AzLineageMetadataEtl.java @@ -31,20 +31,6 @@ public class AzLineageMetadataEtl extends EtlJob { public Long endTimeStamp = null; Connection conn; - /** - * Assume that all job running on one Azkaban instance will running on the same Hadoop instance - * @param azkabanInstanceId - */ - public AzLineageMetadataEtl(int azkabanInstanceId) { - super(azkabanInstanceId, null, 0L); - // for default - try { - setUp(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - /** * Used by backend server * @param appId The application id for the target azkaban server diff --git a/wherehows-etl/src/main/java/metadata/etl/lineage/appworx/AppworxLineageEtl.java b/wherehows-etl/src/main/java/metadata/etl/lineage/appworx/AppworxLineageEtl.java index eb612e4913..ef5601b704 100644 --- a/wherehows-etl/src/main/java/metadata/etl/lineage/appworx/AppworxLineageEtl.java +++ b/wherehows-etl/src/main/java/metadata/etl/lineage/appworx/AppworxLineageEtl.java @@ -26,15 +26,6 @@ public class AppworxLineageEtl extends EtlJob { Connection conn; - public AppworxLineageEtl(int appId, long whExecId) { - super(appId, null, whExecId); - try { - setUp(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - public AppworxLineageEtl(int appId, long whExecId, Properties properties) { super(appId, null, whExecId, properties); try { diff --git a/wherehows-etl/src/main/java/metadata/etl/metadata/DatasetDescriptionEtl.java b/wherehows-etl/src/main/java/metadata/etl/metadata/DatasetDescriptionEtl.java index c64712c783..327fb21698 100644 --- a/wherehows-etl/src/main/java/metadata/etl/metadata/DatasetDescriptionEtl.java +++ b/wherehows-etl/src/main/java/metadata/etl/metadata/DatasetDescriptionEtl.java @@ -29,10 +29,6 @@ import wherehows.common.Constant; public class DatasetDescriptionEtl extends EtlJob { - @Deprecated - public DatasetDescriptionEtl(int dbId, long whExecId) { - super(null, dbId, whExecId); - } public DatasetDescriptionEtl(int dbId, long whExecId, Properties prop) { super(null, dbId, whExecId, prop); diff --git a/wherehows-etl/src/main/java/metadata/etl/ownership/DaliViewOwnerEtl.java b/wherehows-etl/src/main/java/metadata/etl/ownership/DaliViewOwnerEtl.java index 4c032e5df1..28c26ed060 100644 --- a/wherehows-etl/src/main/java/metadata/etl/ownership/DaliViewOwnerEtl.java +++ b/wherehows-etl/src/main/java/metadata/etl/ownership/DaliViewOwnerEtl.java @@ -27,10 +27,6 @@ public class DaliViewOwnerEtl extends EtlJob { public ClassLoader classLoader = getClass().getClassLoader(); protected final Logger logger = LoggerFactory.getLogger(getClass()); - public DaliViewOwnerEtl(int dbId, long whExecId) { - super(null, dbId, whExecId); - } - public DaliViewOwnerEtl(int appId, long whExecId, Properties prop) { super(appId, null, whExecId, prop); } diff --git a/wherehows-etl/src/main/java/metadata/etl/ownership/DatasetOwnerEtl.java b/wherehows-etl/src/main/java/metadata/etl/ownership/DatasetOwnerEtl.java index 96b4763b77..826e92856e 100644 --- a/wherehows-etl/src/main/java/metadata/etl/ownership/DatasetOwnerEtl.java +++ b/wherehows-etl/src/main/java/metadata/etl/ownership/DatasetOwnerEtl.java @@ -39,10 +39,6 @@ import wherehows.common.Constant; * This DatasetOwnerEtl job is extract ownership info from the hive table, transform and store into WhereHows database. */ public class DatasetOwnerEtl extends EtlJob { - @Deprecated - public DatasetOwnerEtl(int dbId, long whExecId) { - super(null, dbId, whExecId); - } public DatasetOwnerEtl(int dbId, long whExecId, Properties prop) { super(null, dbId, whExecId, prop); diff --git a/wherehows-etl/src/main/java/metadata/etl/security/DatasetConfidentialFieldEtl.java b/wherehows-etl/src/main/java/metadata/etl/security/DatasetConfidentialFieldEtl.java index f0a9ac3977..7d19487865 100644 --- a/wherehows-etl/src/main/java/metadata/etl/security/DatasetConfidentialFieldEtl.java +++ b/wherehows-etl/src/main/java/metadata/etl/security/DatasetConfidentialFieldEtl.java @@ -29,10 +29,6 @@ import wherehows.common.Constant; public class DatasetConfidentialFieldEtl extends EtlJob { - @Deprecated - public DatasetConfidentialFieldEtl(int dbId, long whExecId) { - super(null, dbId, whExecId); - } public DatasetConfidentialFieldEtl(int dbId, long whExecId, Properties prop) { super(null, dbId, whExecId, prop); diff --git a/wherehows-etl/src/main/java/metadata/etl/treebuilder/DatasetTreeBuildETL.java b/wherehows-etl/src/main/java/metadata/etl/treebuilder/DatasetTreeBuildETL.java index ffcdb2316b..2f211a8e84 100644 --- a/wherehows-etl/src/main/java/metadata/etl/treebuilder/DatasetTreeBuildETL.java +++ b/wherehows-etl/src/main/java/metadata/etl/treebuilder/DatasetTreeBuildETL.java @@ -20,10 +20,6 @@ import metadata.etl.EtlJob; public class DatasetTreeBuildETL extends EtlJob { - public DatasetTreeBuildETL(int appId, long whExecId) { - super(appId, null, whExecId); - } - public DatasetTreeBuildETL(int appId, long whExecId, Properties properties) { super(appId, null, whExecId, properties); } diff --git a/wherehows-etl/src/test/java/metadata/etl/dataset/hdfs/HdfsMetadataEtlTest.java b/wherehows-etl/src/test/java/metadata/etl/dataset/hdfs/HdfsMetadataEtlTest.java index b5e7eda7be..47c77593ac 100644 --- a/wherehows-etl/src/test/java/metadata/etl/dataset/hdfs/HdfsMetadataEtlTest.java +++ b/wherehows-etl/src/test/java/metadata/etl/dataset/hdfs/HdfsMetadataEtlTest.java @@ -13,6 +13,7 @@ */ package metadata.etl.dataset.hdfs; +import java.util.Properties; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; @@ -24,34 +25,29 @@ public class HdfsMetadataEtlTest { HdfsMetadataEtl ds; @BeforeTest - public void setUp() - throws Exception { - ds = new HdfsMetadataEtl(2, 0L); + public void setUp() throws Exception { + ds = new HdfsMetadataEtl(2, 0L, new Properties()); } @Test(groups = {"needConfig"}) - public void testRun() - throws Exception { + public void testRun() throws Exception { ds.run(); } @Test(groups = {"needConfig"}) - public void testExtract() - throws Exception { + public void testExtract() throws Exception { ds.extract(); //TODO check it copy back the files } @Test(groups = {"needConfig"}) - public void testTransform() - throws Exception { + public void testTransform() throws Exception { ds.transform(); //TODO check it generate the final csv file } @Test(groups = {"needConfig"}) - public void testLoad() - throws Exception { + public void testLoad() throws Exception { ds.load(); } } diff --git a/wherehows-etl/src/test/java/metadata/etl/dataset/hive/HiveTest.java b/wherehows-etl/src/test/java/metadata/etl/dataset/hive/HiveTest.java index 9eb6c1aeb4..a96a289076 100644 --- a/wherehows-etl/src/test/java/metadata/etl/dataset/hive/HiveTest.java +++ b/wherehows-etl/src/test/java/metadata/etl/dataset/hive/HiveTest.java @@ -13,6 +13,7 @@ */ package metadata.etl.dataset.hive; +import java.util.Properties; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; @@ -24,38 +25,32 @@ public class HiveTest { HiveMetadataEtl hm; @BeforeTest - public void setUp() - throws Exception { - hm = new HiveMetadataEtl(3, 0L); + public void setUp() throws Exception { + hm = new HiveMetadataEtl(3, 0L, new Properties()); } @Test - public void extractTest() - throws Exception { + public void extractTest() throws Exception { hm.extract(); // check the json file } @Test - public void transformTest() - throws Exception { + public void transformTest() throws Exception { hm.transform(); // check the csv file } @Test - public void loadTest() - throws Exception { + public void loadTest() throws Exception { hm.load(); // check in database } @Test - public void runTest() - throws Exception { + public void runTest() throws Exception { extractTest(); transformTest(); loadTest(); } - } diff --git a/wherehows-etl/src/test/java/metadata/etl/git/GitMetadataEtlTest.java b/wherehows-etl/src/test/java/metadata/etl/git/GitMetadataEtlTest.java index 0687244975..1af0b47c18 100644 --- a/wherehows-etl/src/test/java/metadata/etl/git/GitMetadataEtlTest.java +++ b/wherehows-etl/src/test/java/metadata/etl/git/GitMetadataEtlTest.java @@ -13,6 +13,7 @@ */ package metadata.etl.git; +import java.util.Properties; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -24,32 +25,27 @@ public class GitMetadataEtlTest { GitMetadataEtl git; @BeforeMethod - public void setUp() - throws Exception { - this.git = new GitMetadataEtl(500, 0L); + public void setUp() throws Exception { + this.git = new GitMetadataEtl(500, 0L, new Properties()); } @Test - public void testExtract() - throws Exception { + public void testExtract() throws Exception { git.extract(); } @Test - public void testTransform() - throws Exception { + public void testTransform() throws Exception { git.transform(); } @Test - public void testLoad() - throws Exception { + public void testLoad() throws Exception { git.load(); } @Test - public void testRun() - throws Exception { + public void testRun() throws Exception { git.run(); } } \ No newline at end of file diff --git a/wherehows-etl/src/test/java/metadata/etl/lineage/LineageTest.java b/wherehows-etl/src/test/java/metadata/etl/lineage/LineageTest.java index 41367210bc..0a76ba6beb 100644 --- a/wherehows-etl/src/test/java/metadata/etl/lineage/LineageTest.java +++ b/wherehows-etl/src/test/java/metadata/etl/lineage/LineageTest.java @@ -28,7 +28,7 @@ public class LineageTest { AzLineageMetadataEtl lm; public LineageTest() { - lm = new AzLineageMetadataEtl(31); + lm = new AzLineageMetadataEtl(31, 0L, properties); properties = lm.prop; } diff --git a/wherehows-etl/src/test/java/metadata/etl/lineage/appworx/AppworxLineageEtlTest.java b/wherehows-etl/src/test/java/metadata/etl/lineage/appworx/AppworxLineageEtlTest.java index d01cc5d95b..396ebcfbf6 100644 --- a/wherehows-etl/src/test/java/metadata/etl/lineage/appworx/AppworxLineageEtlTest.java +++ b/wherehows-etl/src/test/java/metadata/etl/lineage/appworx/AppworxLineageEtlTest.java @@ -13,38 +13,36 @@ */ package metadata.etl.lineage.appworx; +import java.util.Properties; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; -public class AppworxLineageEtlTest{ - AppworxLineageEtl awl; +public class AppworxLineageEtlTest { + AppworxLineageEtl awl; - @BeforeTest - public void setUp() - throws Exception { - awl = new AppworxLineageEtl(3, 0L); - } + @BeforeTest + public void setUp() throws Exception { + awl = new AppworxLineageEtl(3, 0L, new Properties()); + } + @Test(groups = {"needConfig"}) + public void testExtract() throws Exception { + awl.extract(); + } - @Test(groups = {"needConfig"}) - public void testExtract() throws Exception { - awl.extract(); - } + @Test(groups = {"needConfig"}) + public void testTransform() throws Exception { + awl.transform(); + } - @Test(groups = {"needConfig"}) - public void testTransform() throws Exception { - awl.transform(); - } - - @Test(groups = {"needConfig"}) - public void testLoad() throws Exception { - awl.load(); - } - - @Test(groups = {"needConfig"}) - public void testRun() throws Exception { - awl.run(); - } + @Test(groups = {"needConfig"}) + public void testLoad() throws Exception { + awl.load(); + } + @Test(groups = {"needConfig"}) + public void testRun() throws Exception { + awl.run(); + } } diff --git a/wherehows-etl/src/test/java/metadata/etl/metadata/DatasetDescriptionEtlTest.java b/wherehows-etl/src/test/java/metadata/etl/metadata/DatasetDescriptionEtlTest.java index ffafec65da..cc944e828d 100644 --- a/wherehows-etl/src/test/java/metadata/etl/metadata/DatasetDescriptionEtlTest.java +++ b/wherehows-etl/src/test/java/metadata/etl/metadata/DatasetDescriptionEtlTest.java @@ -13,6 +13,7 @@ */ package metadata.etl.metadata; +import java.util.Properties; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; @@ -22,9 +23,8 @@ public class DatasetDescriptionEtlTest { DatasetDescriptionEtl datasetDescriptionEtl; @BeforeTest - public void setUp() - throws Exception { - datasetDescriptionEtl = new DatasetDescriptionEtl(50, 0L); + public void setUp() throws Exception { + datasetDescriptionEtl = new DatasetDescriptionEtl(50, 0L, new Properties()); } /* diff --git a/wherehows-etl/src/test/java/metadata/etl/ownership/DaliViewOwnerEtlTest.java b/wherehows-etl/src/test/java/metadata/etl/ownership/DaliViewOwnerEtlTest.java index f9ddbf3f7c..9e6294db07 100644 --- a/wherehows-etl/src/test/java/metadata/etl/ownership/DaliViewOwnerEtlTest.java +++ b/wherehows-etl/src/test/java/metadata/etl/ownership/DaliViewOwnerEtlTest.java @@ -13,6 +13,7 @@ */ package metadata.etl.ownership; +import java.util.Properties; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -25,32 +26,27 @@ public class DaliViewOwnerEtlTest { DaliViewOwnerEtl dali; @BeforeMethod - public void setUp() - throws Exception { - dali = new DaliViewOwnerEtl(60, 0L); + public void setUp() throws Exception { + dali = new DaliViewOwnerEtl(60, 0L, new Properties()); } @Test - public void testExtract() - throws Exception { + public void testExtract() throws Exception { dali.extract(); } @Test - public void testTransform() - throws Exception { + public void testTransform() throws Exception { dali.transform(); } @Test - public void testLoad() - throws Exception { + public void testLoad() throws Exception { dali.load(); } @Test - public void testRun() - throws Exception { + public void testRun() throws Exception { dali.run(); } } \ No newline at end of file diff --git a/wherehows-etl/src/test/java/metadata/etl/ownership/DatasetOwnerEtlTest.java b/wherehows-etl/src/test/java/metadata/etl/ownership/DatasetOwnerEtlTest.java index 5b245b7fb8..c08ad890ec 100644 --- a/wherehows-etl/src/test/java/metadata/etl/ownership/DatasetOwnerEtlTest.java +++ b/wherehows-etl/src/test/java/metadata/etl/ownership/DatasetOwnerEtlTest.java @@ -13,6 +13,7 @@ */ package metadata.etl.ownership; +import java.util.Properties; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; @@ -25,14 +26,12 @@ public class DatasetOwnerEtlTest { DatasetOwnerEtl doe; @BeforeTest - public void setUp() - throws Exception { - doe = new DatasetOwnerEtl(21, 0L); + public void setUp() throws Exception { + doe = new DatasetOwnerEtl(21, 0L, new Properties()); } @Test(groups = {"needConfig"}) - public void testExtract() - throws Exception { + public void testExtract() throws Exception { doe.extract(); } diff --git a/wherehows-etl/src/test/java/metadata/etl/security/DatasetConfidentialFieldEtlTest.java b/wherehows-etl/src/test/java/metadata/etl/security/DatasetConfidentialFieldEtlTest.java index d6b3d0ca33..a25de426cd 100644 --- a/wherehows-etl/src/test/java/metadata/etl/security/DatasetConfidentialFieldEtlTest.java +++ b/wherehows-etl/src/test/java/metadata/etl/security/DatasetConfidentialFieldEtlTest.java @@ -13,6 +13,7 @@ */ package metadata.etl.security; +import java.util.Properties; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; @@ -22,14 +23,12 @@ public class DatasetConfidentialFieldEtlTest { DatasetConfidentialFieldEtl confidentialFieldEtl; @BeforeTest - public void setUp() - throws Exception { - confidentialFieldEtl = new DatasetConfidentialFieldEtl(49, 0L); + public void setUp() throws Exception { + confidentialFieldEtl = new DatasetConfidentialFieldEtl(49, 0L, new Properties()); } @Test(groups = {"needConfig"}) - public void testExtract() - throws Exception { + public void testExtract() throws Exception { confidentialFieldEtl.extract(); } diff --git a/wherehows-etl/src/test/java/metadata/etl/treebuilder/DatasetTreeBuildETLTest.java b/wherehows-etl/src/test/java/metadata/etl/treebuilder/DatasetTreeBuildETLTest.java index 665bc4bc06..53c5e494e0 100644 --- a/wherehows-etl/src/test/java/metadata/etl/treebuilder/DatasetTreeBuildETLTest.java +++ b/wherehows-etl/src/test/java/metadata/etl/treebuilder/DatasetTreeBuildETLTest.java @@ -13,18 +13,17 @@ */ package metadata.etl.treebuilder; +import java.util.Properties; import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; - -public class DatasetTreeBuildETLTest{ +public class DatasetTreeBuildETLTest { DatasetTreeBuildETL datasetTreeBuildETL; @BeforeTest - public void setUp() - throws Exception { - datasetTreeBuildETL = new DatasetTreeBuildETL(40, 0L); + public void setUp() throws Exception { + datasetTreeBuildETL = new DatasetTreeBuildETL(40, 0L, new Properties()); } @Test(groups = {"needConfig"})