yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-19 06:38:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #128 from rkluszczynski/fixTreeBuildersWithES

Browse Source 
Fixing tree builder scripts when ES is not used
This commit is contained in:
jerrybai2009 2016-05-24 15:19:45 -07:00
commit c1e8baf1c5
2 changed files with 32 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
metadata-etl/src/main/resources/jython/DatasetTreeBuilder.py
View File

@ -12,14 +12,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# #
import sys
from com.ziclix.python.sql import zxJDBC
from wherehows.common import Constant
from ElasticSearchIndex import ElasticSearchIndex
from datetime import datetime
import calendar import calendar
import json import json
import shutil import shutil
import sys
from com.ziclix.python.sql import zxJDBC
from wherehows.common import Constant
from datetime import datetime
from jython.ElasticSearchIndex import ElasticSearchIndex
class DatasetTreeBuilder: class DatasetTreeBuilder:
@ -79,10 +80,16 @@ class DatasetTreeBuilder:
self.write_to_file() self.write_to_file()
def saveTreeInElasticSearchIfApplicable(args):
es_url = args.get(Constant.WH_ELASTICSEARCH_URL_KEY, None)
es_port = args.get(Constant.WH_ELASTICSEARCH_PORT_KEY, None)
if es_url and es_port:
esi = ElasticSearchIndex(args)
d = datetime.utcnow()
unixtime = calendar.timegm(d.utctimetuple())
esi.update_dataset(unixtime)
if __name__ == "__main__": if __name__ == "__main__":
d = DatasetTreeBuilder(sys.argv[1]) d = DatasetTreeBuilder(sys.argv[1])
d.run() d.run()
esi = ElasticSearchIndex(sys.argv[1]) saveTreeInElasticSearchIfApplicable(sys.argv[1])
d = datetime.utcnow()
unixtime = calendar.timegm(d.utctimetuple())
esi.update_dataset(unixtime)

25
metadata-etl/src/main/resources/jython/FlowTreeBuilder.py
View File

@ -12,14 +12,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# #
import sys
from com.ziclix.python.sql import zxJDBC
from wherehows.common import Constant
from ElasticSearchIndex import ElasticSearchIndex
from datetime import datetime
import calendar import calendar
import json import json
import shutil import shutil
import sys
from com.ziclix.python.sql import zxJDBC
from datetime import datetime
from wherehows.common import Constant
from jython.ElasticSearchIndex import ElasticSearchIndex
class FlowTreeBuilder: class FlowTreeBuilder:
@ -82,10 +83,16 @@ class FlowTreeBuilder:
self.write_to_file() self.write_to_file()
def saveTreeInElasticSearchIfApplicable(args):
es_url = args.get(Constant.WH_ELASTICSEARCH_URL_KEY, None)
es_port = args.get(Constant.WH_ELASTICSEARCH_PORT_KEY, None)
if es_url and es_port:
esi = ElasticSearchIndex(args)
d = datetime.utcnow()
unixtime = calendar.timegm(d.utctimetuple())
esi.update_flow_jobs(unixtime)
if __name__ == "__main__": if __name__ == "__main__":
ftb = FlowTreeBuilder(sys.argv[1]) ftb = FlowTreeBuilder(sys.argv[1])
ftb.run() ftb.run()
esi = ElasticSearchIndex(sys.argv[1]) saveTreeInElasticSearchIfApplicable(sys.argv[1])
d = datetime.utcnow()
unixtime = calendar.timegm(d.utctimetuple())
esi.update_flow_jobs(unixtime)