From ee1e36219aabc3e6fb66eaed3d79d991d7a73890 Mon Sep 17 00:00:00 2001 From: Rafal Kluszczynski Date: Tue, 24 May 2016 16:04:31 +0200 Subject: [PATCH 1/2] fix: generate tree also when elastic is not configured --- .../resources/jython/DatasetTreeBuilder.py | 25 ++++++++++++------- .../main/resources/jython/FlowTreeBuilder.py | 25 ++++++++++++------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/metadata-etl/src/main/resources/jython/DatasetTreeBuilder.py b/metadata-etl/src/main/resources/jython/DatasetTreeBuilder.py index 57631d636c..55d62b1606 100644 --- a/metadata-etl/src/main/resources/jython/DatasetTreeBuilder.py +++ b/metadata-etl/src/main/resources/jython/DatasetTreeBuilder.py @@ -12,14 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -import sys -from com.ziclix.python.sql import zxJDBC -from wherehows.common import Constant -from ElasticSearchIndex import ElasticSearchIndex -from datetime import datetime import calendar import json import shutil +import sys +from com.ziclix.python.sql import zxJDBC +from wherehows.common import Constant +from datetime import datetime + +from jython.ElasticSearchIndex import ElasticSearchIndex class DatasetTreeBuilder: @@ -79,10 +80,16 @@ class DatasetTreeBuilder: self.write_to_file() +def saveTreeInElasticSearchIfApplicable(args): + es_url = args.get(Constant.WH_ELASTICSEARCH_URL_KEY, None) + es_port = args.get(Constant.WH_ELASTICSEARCH_PORT_KEY, None) + if es_url is not None and es_port is not None: + esi = ElasticSearchIndex(args) + d = datetime.utcnow() + unixtime = calendar.timegm(d.utctimetuple()) + esi.update_dataset(unixtime) + if __name__ == "__main__": d = DatasetTreeBuilder(sys.argv[1]) d.run() - esi = ElasticSearchIndex(sys.argv[1]) - d = datetime.utcnow() - unixtime = calendar.timegm(d.utctimetuple()) - esi.update_dataset(unixtime) + saveTreeInElasticSearchIfApplicable(sys.argv[1]) diff --git a/metadata-etl/src/main/resources/jython/FlowTreeBuilder.py b/metadata-etl/src/main/resources/jython/FlowTreeBuilder.py index 91fa143259..9d80945f5d 100644 --- a/metadata-etl/src/main/resources/jython/FlowTreeBuilder.py +++ b/metadata-etl/src/main/resources/jython/FlowTreeBuilder.py @@ -12,14 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -import sys -from com.ziclix.python.sql import zxJDBC -from wherehows.common import Constant -from ElasticSearchIndex import ElasticSearchIndex -from datetime import datetime import calendar import json import shutil +import sys +from com.ziclix.python.sql import zxJDBC +from datetime import datetime +from wherehows.common import Constant + +from jython.ElasticSearchIndex import ElasticSearchIndex class FlowTreeBuilder: @@ -82,10 +83,16 @@ class FlowTreeBuilder: self.write_to_file() +def saveTreeInElasticSearchIfApplicable(args): + es_url = args.get(Constant.WH_ELASTICSEARCH_URL_KEY, None) + es_port = args.get(Constant.WH_ELASTICSEARCH_PORT_KEY, None) + if es_url is not None and es_port is not None: + esi = ElasticSearchIndex(args) + d = datetime.utcnow() + unixtime = calendar.timegm(d.utctimetuple()) + esi.update_flow_jobs(unixtime) + if __name__ == "__main__": ftb = FlowTreeBuilder(sys.argv[1]) ftb.run() - esi = ElasticSearchIndex(sys.argv[1]) - d = datetime.utcnow() - unixtime = calendar.timegm(d.utctimetuple()) - esi.update_flow_jobs(unixtime) + saveTreeInElasticSearchIfApplicable(sys.argv[1]) From 457b0c0a2eb8fe93aa4d67b23145b66ff6abb6d9 Mon Sep 17 00:00:00 2001 From: Rafal Kluszczynski Date: Tue, 24 May 2016 22:39:59 +0200 Subject: [PATCH 2/2] refactor: apply code review remarks --- metadata-etl/src/main/resources/jython/DatasetTreeBuilder.py | 2 +- metadata-etl/src/main/resources/jython/FlowTreeBuilder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-etl/src/main/resources/jython/DatasetTreeBuilder.py b/metadata-etl/src/main/resources/jython/DatasetTreeBuilder.py index 55d62b1606..c8ae97b42e 100644 --- a/metadata-etl/src/main/resources/jython/DatasetTreeBuilder.py +++ b/metadata-etl/src/main/resources/jython/DatasetTreeBuilder.py @@ -83,7 +83,7 @@ class DatasetTreeBuilder: def saveTreeInElasticSearchIfApplicable(args): es_url = args.get(Constant.WH_ELASTICSEARCH_URL_KEY, None) es_port = args.get(Constant.WH_ELASTICSEARCH_PORT_KEY, None) - if es_url is not None and es_port is not None: + if es_url and es_port: esi = ElasticSearchIndex(args) d = datetime.utcnow() unixtime = calendar.timegm(d.utctimetuple()) diff --git a/metadata-etl/src/main/resources/jython/FlowTreeBuilder.py b/metadata-etl/src/main/resources/jython/FlowTreeBuilder.py index 9d80945f5d..e40eb61655 100644 --- a/metadata-etl/src/main/resources/jython/FlowTreeBuilder.py +++ b/metadata-etl/src/main/resources/jython/FlowTreeBuilder.py @@ -86,7 +86,7 @@ class FlowTreeBuilder: def saveTreeInElasticSearchIfApplicable(args): es_url = args.get(Constant.WH_ELASTICSEARCH_URL_KEY, None) es_port = args.get(Constant.WH_ELASTICSEARCH_PORT_KEY, None) - if es_url is not None and es_port is not None: + if es_url and es_port: esi = ElasticSearchIndex(args) d = datetime.utcnow() unixtime = calendar.timegm(d.utctimetuple())